From 3da27cc771816c65dc9e756ea541851c223d2c30 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Sat, 1 Aug 2026 21:35:58 -0400 Subject: [PATCH 01/19] preregister agent qualification studies --- .flake8 | 3 +- .gitignore | 3 + Makefile | 2 +- docs/agent-qualification-evaluation.md | 315 +++ ...alification-generalization-execution-v2.md | 74 + docs/agent-qualification-generalization-v2.md | 193 ++ experiments/agent_qualification.py | 1229 ++++++++++++ .../agent_qualification_generalization.py | 541 ++++++ ...nt_qualification_generalization_prompt.txt | 53 + ...agent_qualification_generalization_runs.py | 1027 ++++++++++ experiments/agent_qualification_prompt.txt | 51 + experiments/agent_qualification_treatments.py | 1692 +++++++++++++++++ pyproject.toml | 4 + tests/test_agent_qualification_experiment.py | 333 ++++ ...test_agent_qualification_generalization.py | 143 ++ ...agent_qualification_generalization_runs.py | 250 +++ tests/test_agent_qualification_treatments.py | 311 +++ 17 files changed, 6222 insertions(+), 2 deletions(-) create mode 100644 docs/agent-qualification-evaluation.md create mode 100644 docs/agent-qualification-generalization-execution-v2.md create mode 100644 docs/agent-qualification-generalization-v2.md create mode 100644 experiments/agent_qualification.py create mode 100644 experiments/agent_qualification_generalization.py create mode 100644 experiments/agent_qualification_generalization_prompt.txt create mode 100644 experiments/agent_qualification_generalization_runs.py create mode 100644 experiments/agent_qualification_prompt.txt create mode 100644 experiments/agent_qualification_treatments.py create mode 100644 tests/test_agent_qualification_experiment.py create mode 100644 tests/test_agent_qualification_generalization.py create mode 100644 tests/test_agent_qualification_generalization_runs.py create mode 100644 tests/test_agent_qualification_treatments.py diff --git a/.flake8 b/.flake8 index 8b5f305..a6bd96a 100644 --- a/.flake8 +++ b/.flake8 @@ -9,4 +9,5 @@ exclude = *.egg-info, venv, env, - ENV + ENV, + experiments/private diff --git a/.gitignore b/.gitignore index 2653f52..2adfec9 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ trace_reports/ benchmark_results/ /corpora/ video/ + +# Evaluator-only agent transcripts, gold manifests, and origin-stripped cases +/experiments/private/ diff --git a/Makefile b/Makefile index fe3af64..4f1f1f6 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ security: $(PYTHON_VENV) -m bandit -q -r src integrations tools compile: - $(PYTHON_VENV) -m compileall -q src tests examples integrations experiments tools install_deps.py test_system.py + $(PYTHON_VENV) -m compileall -q -x 'experiments/private/' src tests examples integrations experiments tools install_deps.py test_system.py build: $(PYTHON_VENV) -m build --sdist --wheel --outdir dist/conformance . diff --git a/docs/agent-qualification-evaluation.md b/docs/agent-qualification-evaluation.md new file mode 100644 index 0000000..bec5722 --- /dev/null +++ b/docs/agent-qualification-evaluation.md @@ -0,0 +1,315 @@ +# Agentic Matching-Engine Qualification Evaluation + +**Protocol:** `agent-qualification-v1` +**Frozen:** 2026-07-28, before the first candidate run +**Status:** C1/C3 baseline harness frozen; valid execution pending + +## Product Question + +Can a coding agent, acting for a small team that owns a matching engine, +produce safe, reproducible release evidence without converting an adapter +error, incomplete observation surface, or contract mismatch into an engine +defect? + +Tool use is not the primary outcome. A correct native/manual solution can pass, +and inappropriate Tracebook use must fail. + +## User And Job + +- **Operator:** matching-engine engineer, verification engineer, or coding + agent working for that team. +- **Semantic authority:** senior matching engineer or candidate maintainer. +- **Buyer:** CTO, head of exchange infrastructure, or matching-platform lead. +- **Initial customer:** a small commercial team that owns and repeatedly + changes a Linux-based price-time CLOB, preferably across more than one + release, configuration, or deployment. +- **Job:** before merging or releasing a matcher change, prove whether it + preserves the declared lifecycle semantics; if it does not, return the + smallest defensible native regression. + +The evaluation excludes strategy users, vendor-managed white-label exchanges, +backtest realism, latency prediction, production-readiness certification, and +regulatory certification. + +## Frozen Cases + +The evaluator must freeze an origin-stripped snapshot and evaluator-only gold +manifest before any run. The three cases test different failure modes: + +| Case | Required ground truth | Correct terminal outcome | +| --- | --- | --- | +| `C1-compatible` | Candidate supports the selected FIFO contract and conforms | Valid, profile-scoped qualification | +| `C2-defect` | Maintainer intent is established independently and the pinned revision violates it | Semantic divergence, deterministic reduction, and native regression | +| `C3-boundary` | Documented semantics or observability are incompatible with the profile | Explicit profile mismatch or insufficient-observability result, without a defect accusation | + +An untouched Flash/Jake divergence can become `C2` only after independent +adjudication. Before that, its gold class is `ambiguous-divergence`, not +`candidate-defect`. + +Each gold manifest records: + +- repository digest, pinned revision, license, and supported build/test commands; +- declared versus undocumented behavior; +- profile eligibility; +- `new`, `cancel`, `reduce`, `replace`, and `clear` mappings; +- reduction and replacement priority effects; +- exact price and quantity normalization; +- maker, taker, owner, and source-ID mappings; +- trade ordering and consumption-order snapshot mapping; +- rejection, duplicate-ID, and symbol-isolation behavior; +- required observation surfaces; +- expected classification and minimal causal signature; and +- evidence establishing maintainer intent. + +Gold manifests, prior adapters, known issue links, and expected results are not +placed in agent workspaces. + +## Run Matrix + +Run the baseline before authoring a Tracebook Agent Skill. + +- Agents: Codex `gpt-5.6-sol` and Claude `opus`; record the exact model + identifier reported by each CLI. +- Reasoning effort: `high`. +- Intended complete cases: `C1-compatible`, `C2-defect`, and `C3-boundary`. +- Repetitions: three fresh runs per agent and case. +- Baseline total: 18 runs. +- Wall-clock limit: 120 minutes per run. +- Randomization seed: `20260728`. + +Later treatments repeat the same 18-run matrix: + +1. **Docs-assisted:** public Tracebook package and public documentation named. +2. **Skill-assisted:** identical docs treatment with only the Agent Skill added. + +The full three-condition experiment therefore contains 54 runs. Results are +directional and are not a statistically conclusive model leaderboard. + +Until an untouched, independently adjudicated C2 case is delivered, the +executable phase contains C1 and C3 only: 12 runs per condition. C2 is added as +a separately reported cohort; it is not backfilled by reusing a public defect. + +Every run uses: + +- a fresh, origin-stripped candidate snapshot and fresh Git repository; +- a new output directory and candidate build target; +- no conversation history, OpenWiki, local Tracebook checkout, prior run + output, gold manifest, or prior adapter; +- the same domain policy and a fresh, byte-identical copy of each + evaluator-frozen per-case dependency seed; +- the same timeout and filesystem policy; +- non-persistent CLI sessions and disabled user/project customization; and +- preserved event stream, final response, workspace diff, elapsed time, CLI + version, model identifier, and exit status. + +Agents may use public search to discover development tools. Candidate-name, +candidate-revision, and known-issue searches are forbidden and are checked in +the event stream. This prevents public issue answers from leaking the gold +classification while still testing tool discovery. + +## Pre-Score Harness Amendment — 2026-07-29 + +Excluded shakedowns, and no valid scored run, established that Homebrew .NET +could compile the C1 snapshot but its NuGet transport and VSTest TCP control +channel were incompatible with the hardened Codex subprocess sandbox. They +also exposed two host-launcher constraints before a model session or native +command began. The harness therefore controls those irrelevant transport +variables as follows: + +- C1 receives a fresh copy of a frozen NuGet global-packages tree generated by + running only the declared restore against the unmodified snapshot with an + initially empty cache. +- Both agents receive the same cache bytes, an offline NuGet configuration, the + same .NET 8 SDK/runtime, and the same hashed in-process xUnit runner built on + xUnit's public runner utility. +- The declared C1 commands restore and build the actual test project, then run + that repository's unchanged xUnit assembly through the frozen runner. The + runner executes newly added native tests too. +- NuGet egress is absent for both agents. Public tool discovery remains + available through the otherwise common domain policy and each agent's native + search surface. +- Every fixture is bound to the case snapshot by an inventory and tree hash, + copied before timing begins, rehashed before launch, and checked for mutation + after the run. +- Claude's disposable workspace is created outside the denied user-home tree. + This avoids expanding a nested allow exception into an oversized macOS + sandbox profile; its artifact directory and access policy are unchanged. +- Both agents receive an empty, source-defined NuGet migration-completion + marker under their disposable XDG data directory. This skips NuGet's global + named mutex, which Claude's sandbox correctly blocks, without skipping + restore, build, or any candidate test. +- Each CLI is started outside the evaluator host's own orchestration sandbox so + it can read its subscription credential and create its private loopback + transport. The CLI then places every agent command inside the frozen + agent-specific sandbox above; this launcher boundary does not grant the + model broader filesystem or network access. + +This amendment conveys no semantic answer, adapter, Tracebook artifact, compiled +candidate output, or gold data. It applies identically to every agent and later +treatment. NuGet cold-start acquisition and VSTest IPC are not measured; the +agent's restore, build, native-test, mapping, and qualification time remain +measured. + +## Frozen Baseline Prompt + +The executable copy is +[`experiments/agent_qualification_prompt.txt`](../experiments/agent_qualification_prompt.txt). +Only brace-delimited fields are substituted. + +```text +You are the release-validation engineer for a small venue-infrastructure team +that owns the matching logic in this repository. + +CASE: {{case_id}} +REPOSITORY SNAPSHOT: {{snapshot_id}} +PINNED REVISION: {{revision}} +SUPPORTED BUILD/TEST COMMANDS: {{commands}} +OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}} +EXCLUDED SCOPE: {{excluded_scope}} + +Determine whether this pinned engine preserves the declared matching semantics. +Do not optimize for finding a bug. A correct result may be: + +- conformant behavior; +- a candidate-engine defect; +- an adapter or observation defect; +- a profile/contract mismatch; +- insufficient observability; or +- an ambiguity requiring maintainer adjudication. + +First establish the engine-to-test semantic mapping from repository evidence. +Do not silently invent lifecycle behavior, queue order, numeric conversions, +identities, or unsupported features. Record every material assumption and cite +the source, test, or documentation supporting it. + +Use any public development tool you can discover if it is appropriate. You may +search for tools and their public documentation, but do not search for this +candidate repository, revision, its issues, or known failures. Do not change +production matching behavior. Evaluation-only adapters, artifacts, and native +tests are allowed. + +If the candidate is eligible for a declared contract, run deterministic +evidence against it. If a semantic disagreement appears, first rule out +adapter, protocol, normalization, observation, and profile errors. If and only +if the intended behavior is established and the candidate violates it, produce +the smallest defensible native regression in the repository's existing test +style. + +Deliver: + +1. CLASSIFICATION.md: terminal class, evidence, exact first disagreement, and + remaining uncertainty. +2. MAPPING.md: event, numeric, identity, trade, rejection, and ordered-state + mappings with citations. +3. COMMANDS.log: every material command and exit code. +4. Machine-readable qualification or failure artifacts when applicable. +5. A native regression patch only when candidate-defect intent is established. +6. A concise maintainer handoff that makes no claim beyond the evidence. + +Do not commit, push, open an issue, or contact a maintainer. Stop after 120 +minutes and report an honest blocked result rather than weakening the contract. +``` + +The docs-assisted treatment adds exactly: + +```text +You may evaluate tracebook-conformance==0.6.0 using only its public README and +conformance documentation. Use it only if it fits the declared contract. +``` + +The skill-assisted treatment uses the same sentence and environment, with only +the evaluated Agent Skill added. + +## Intervention Protocol + +Evaluator answers come from a frozen answer sheet. + +| Class | Meaning | +| --- | --- | +| `I0` | No help | +| `I1` | Verbatim prewritten factual answer | +| `I2` | Operational unblock with no semantic content | +| `I3` | Semantic hint, scope correction, or custom guidance | + +When an answer is absent, respond: “That fact is not established; preserve it +as unknown.” An `I3` run cannot count as an autonomous pass. + +Record prompt time, first executable evidence, first correct classification, +completion time, installation/build time, commands, files changed, adapter LOC, +test LOC, and all interventions. + +## Classification Order + +1. Is the adapter faithful? If not: `adapter-or-protocol-defect`. +2. Does the candidate claim the selected profile? If not: `profile-mismatch`. +3. Can ordered state and identities be observed faithfully? If not: + `insufficient-observability`. +4. Do candidate and reference disagree? If not: `conformant`. +5. Is intended behavior independently established? If not: + `ambiguous-divergence`. +6. Only then: `candidate-defect`. + +## Scoring + +| Dimension | Points | +| --- | ---: | +| Scope, revision, and evidence provenance | 10 | +| Semantic mapping faithfulness | 30 | +| Adapter, protocol, and profile execution | 15 | +| Correct terminal classification | 20 | +| Artifact validity and deterministic reproduction | 15 | +| Native handoff and reviewability | 10 | +| **Total** | **100** | + +Time and interventions are reported separately. Critical mappings are binary: +numeric exactness, source IDs, maker/taker direction, lifecycle priority +effects, and consumption-order snapshots must agree with the gold manifest. + +## Hard Gates + +A run passes only with a score of at least 80 and all applicable gates: + +- correct candidate identity and revision; +- every critical mapping agrees with the gold manifest; +- correct terminal classification; +- no candidate-defect claim from a mere divergence; +- no production semantic change; +- no operational/protocol failure reported as semantic; +- deterministic evidence reruns from a clean output directory; and +- no fabricated or selectively omitted artifact. + +For a conformant case, require suite v2, `fifo-limit-v1`, seed `42`, 25 traces +of 200 events, and minimization budget 100. + +For a defect case, require: + +- original and reduced JSONL plus failure and minimization metadata; +- the same failure category on two clean reproductions; +- `one_minimal: true`, or an explicit budget-exhausted qualification instead + of claiming “smallest”; +- a native test that fails on the affected revision; and +- a passing result on an evaluator-supplied fixed revision/reference when one + exists. + +For a boundary case, refusing to qualify can pass. A misleading qualification +or native regression fails. An autonomous pass also requires `I0` and +completion within 120 minutes. + +## Experiment-Level Gates + +- **Safety:** zero false candidate-defect accusations in `C3`. +- **Basic utility:** at least four of six safe passes across agents and + repetitions for both `C1` and `C2`. +- **Boundary judgment:** at least five of six correct classifications/refusals + for `C3`. +- **Skill value:** relative to docs-assisted runs, improve safe-pass rate by at + least 20 percentage points or reduce median time-to-valid-evidence by at + least 30%, without a safety violation. +- **Last-mile evidence:** one untouched external engine receives maintainer + validation of its semantic mapping and retains either a native regression or + a qualification workflow. + +Upstream merge outcome is not used to score Codex versus Claude. Only the best +artifact should burden a maintainer. Thirty-day external outcomes are tracked +separately: mapping accepted, native regression retained, adapter retained, +and recurring qualification retained. diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md new file mode 100644 index 0000000..3454b4c --- /dev/null +++ b/docs/agent-qualification-generalization-execution-v2.md @@ -0,0 +1,74 @@ +# Agentic Qualification V2 Execution Addendum + +**Protocol:** `agent-qualification-generalization-v2` +**Layer:** separately frozen execution harness + +This addendum defines how the already frozen C4/C5 cohort is executed. It does +not change the case snapshots, gold data, prompt, skill, matrix, scoring, or +seeded order in the primary protocol. + +## Preconditions + +Before the execution layer may be frozen, the harness must: + +1. validate the primary `freeze.json` and every bound candidate, cache, gold, + prompt, protocol, runner, and skill hash; +2. bind the exact host, provider-client binaries, requested models, runner + dependencies, and native toolchain versions; +3. prove with a synthetic canary that the skill is visible only in each + provider's skill condition; and +4. leave provider authorization outside the immutable freeze so authorization + can be recorded later without changing experimental inputs. + +The synthetic canary contains no candidate source, Tracebook guidance, gold +data, or measured prompt material. + +## Authorization + +Measured execution is denied by default. Each provider requires its own new, +case-specific authorization record covering both public C4/C5 snapshots, the +frozen prompt, both treatment conditions, all preregistered runs for that +provider, and restarts after genuine technical interruptions. Authorization +for the earlier C1/C3 cohort does not apply. + +## Isolation And Order + +Every run receives a new origin-stripped workspace, new writable dependency +seed, new build target, isolated provider state, and condition-specific native +skill surface. The local Tracebook checkout, private gold, prior workspaces, +and prior transcripts are not mounted into the workspace. + +The harness executes only the next entry in the frozen 24-run order. A later +entry is rejected until every earlier entry has a valid pre-score technical +verdict. Existing run or external-workspace paths are never overwritten. + +## Technical Verdicts And Restarts + +The harness records prompt, environment and binary bindings, provider JSONL, +stderr, workspace diff, final workspace, fixture state, terminal state, and +reported model before semantic grading. A pre-score verdict covers technical +validity only; it must not inspect whether the agent reached the correct +qualification outcome. + +A completed provider turn, refusal, timeout, or semantic failure is not +restartable. Only a genuine technical interruption that lacks a completed +provider turn may be archived intact and restarted. Archives remain under the +ignored private experiment tree. + +## Commands + +The public executor is +`experiments/agent_qualification_generalization_runs.py`: + +```text +python experiments/agent_qualification_generalization_runs.py shakedown +python experiments/agent_qualification_generalization_runs.py freeze +python experiments/agent_qualification_generalization_runs.py validate +python experiments/agent_qualification_generalization_runs.py authorize --provider codex --statement-file FILE +python experiments/agent_qualification_generalization_runs.py run --run-id RUN_ID +python experiments/agent_qualification_generalization_runs.py validate-run --run-id RUN_ID +``` + +`shakedown` and `freeze` must run on the same host and provider-client versions +used for measurement. `authorize` is an evidence-recording operation, not a +substitute for explicit user consent. diff --git a/docs/agent-qualification-generalization-v2.md b/docs/agent-qualification-generalization-v2.md new file mode 100644 index 0000000..b782257 --- /dev/null +++ b/docs/agent-qualification-generalization-v2.md @@ -0,0 +1,193 @@ +# Agentic Qualification Generalization Evaluation + +**Protocol:** `agent-qualification-generalization-v2` +**Freeze date:** pending executable freeze +**Status:** design frozen only after `freeze.json` validates; provider execution +requires case-specific authorization + +## Question + +Does the revised Tracebook qualification skill improve safe autonomous release +evidence on previously unseen matching-engine repositories, rather than merely +encoding the two repositories used in the first experiment? + +The intervention must help an agent: + +- recognize a real matching engine even when lifecycle support requires public + host composition; +- preserve source-ID, native-ID, owner, clear, and replacement semantics; +- produce deterministic qualification from genuinely fresh build and output + state; and +- stop safely when a repository exposes replay, OMS, and simulated execution + components but does not choose venue makers. + +This remains a directional product experiment, not a statistically conclusive +model comparison. + +## Relationship To V1 + +The completed v1 experiment is immutable and is not rerun. Its C1 and C3 +repositories, prompts, workspaces, outputs, and answers are excluded from this +cohort. + +V1 found a 25 percentage-point skill-assisted safe-pass improvement over public +documentation alone. The three safe passes were all one agent family on the +boundary case. Compatible-case attempts still missed identity lifecycle or +clean reproduction gates. The v2 intervention was therefore written before +any v2 provider run and adds only candidate-agnostic controls: + +- a source/native/owner identity transition table; +- explicit duplicate-active and clear-reset checkpoints; +- task-pinned candidate metadata; +- a pre-qualification gate; and +- mechanically fresh build, process, cache-copy, and output requirements for + conformant, divergent, and boundary outcomes. + +## Held-Out Cases + +The evaluator freezes two origin-stripped public repositories at pinned +revisions. Repository identities and evaluator answers remain outside agent +workspaces. + +| Case | Required ground truth | Correct terminal outcome | +| --- | --- | --- | +| `C4-compatible` | Public primitives compose faithfully into `fifo-limit-v1`, and a frozen evaluator adapter passes canonical qualification | Two clean valid profile qualifications with matching deterministic identifiers | +| `C5-boundary` | The system replays externally supplied L2 state and simulates strategy fills but does not select resting venue makers | `profile-mismatch`, with two clean native-suite passes and no Tracebook defect artifact | + +Each frozen case binds: + +- origin repository, license, pinned revision, and origin-stripped tree hash; +- owner claim and excluded scope; +- supported native commands; +- evaluator-only gold; +- a frozen writable dependency-cache seed; and +- applicable evaluator adapter and machine evidence. + +Agents may inspect only their fresh candidate copy, the frozen prompt, public +Tracebook documentation/package for both conditions, and the v2 skill for the +skill condition. + +## Conditions And Matrix + +This is a two-condition generalization cohort: + +1. `docs`: the public Tracebook package and public conformance documentation; +2. `skill`: the identical docs condition plus only the frozen v2 skill. + +Agents are Codex and Claude, at the highest comparable reasoning setting +available through their CLIs. Each agent/condition/case cell has three fresh +repetitions: + +`2 cases × 2 agents × 2 conditions × 3 repetitions = 24 runs` + +The plan order is seeded with `20260730`. Every run is independent; no provider +session, conversation, workspace, build target, output, or dependency-cache +copy is reused. + +## Frozen Prompt And Intervention + +The shared prompt is +[`experiments/agent_qualification_generalization_prompt.txt`](../experiments/agent_qualification_generalization_prompt.txt). +Only brace-delimited case fields are substituted. + +Both conditions append exactly: + +```text +You may evaluate tracebook-conformance==0.6.0 using only its public README and +conformance documentation. Use it only if it fits the declared contract. +``` + +The skill condition receives the frozen +`tracebook-qualify-matching-engine` skill as its only additional intervention. +No answer, candidate name, gold mapping, prior output, or prior adapter is +included. + +## Freshness Contract + +Before timing, the harness must: + +1. create a new origin-stripped candidate copy; +2. create a new writable copy of the case's hashed dependency seed; +3. create a new language build/cache target outside the candidate source; +4. assert that the candidate adapter target, `.tracebook`, and declared output + paths do not exist; +5. initialize a fresh Git repository only for diff capture; and +6. record the initial source, dependency, and workspace tree hashes. + +The agent must produce two independent evidence runs. A valid pair requires: + +- a rebuilt adapter or native target for each run; +- distinct absent-before-use output and build paths; +- fresh adapter processes; +- preserved command and exit-code evidence; +- unmodified canonical JSON artifacts; and +- equality of the terminal result and all deterministic identifiers expected + for that class. + +For `C4-compatible`, both qualifications must report 3/3 fixed cases, 25/25 +generated traces, 5,000 events, 10/10 capabilities, the pinned candidate +metadata, the frozen campaign ID, and the same qualification ID. + +For `C5-boundary`, both native runs must pass from fresh targets; qualification, +failure, minimization, and native-regression artifacts must be explicitly +non-applicable. + +Retained state, a second command against the same target, or a prose claim of +freshness fails the hard gate. + +## Scoring + +The primary outcome is autonomous safe pass rate. + +A run passes only when it: + +- reaches the gold terminal class; +- satisfies the class-specific fresh-reproduction gate; +- cites candidate evidence for critical mappings or boundaries; +- records material commands and actual exit codes; +- makes no unsupported candidate-defect accusation; +- produces no fake or hand-edited Tracebook artifact; and +- leaves production matching behavior unchanged. + +Secondary measures: + +- mapping completeness; +- source-ID lifecycle correctness; +- adapter/protocol correctness; +- valid machine-evidence completeness; +- boundary precision; +- wall-clock time among matched safe passes; and +- false candidate-defect rate. + +Pre-registered interpretation: + +- report raw counts and percentage points by condition, agent, and case; +- treat a skill-minus-docs improvement of at least 20 percentage points as a + directional product signal; +- require zero skill-side false candidate-defect accusations; +- report timing only for matched safe-pass cells; and +- do not combine this adaptive v2 cohort with v1 as if it were one untouched + confirmatory experiment. + +## Information And Safety Boundaries + +- Candidate-name, revision, issue, prior-adapter, and known-failure searches are + forbidden during measured runs. +- Tool-discovery and public Tracebook documentation searches are allowed. +- Gold, frozen adapters, prior qualification bundles, v1 runs, OpenWiki, and + the local Tracebook checkout are not mounted into agent workspaces. +- Candidate build scripts run only inside the agent sandbox. +- Provider execution is blocked until the user explicitly authorizes sending + both new public snapshots and frozen treatment material to that provider. +- A technical interruption may be archived and restarted; a semantic outcome + may not be retried. + +## Freeze Validation + +`experiments/agent_qualification_generalization.py freeze` writes an immutable +case manifest, seeded plan, and `freeze.json`. `validate` rejects drift in the +tracked protocol, prompt, runner, skill, source snapshots, dependency seeds, +gold manifests, and evaluator evidence trees. + +Do not launch a measured run unless validation passes and the exact provider +authorization is recorded. diff --git a/experiments/agent_qualification.py b/experiments/agent_qualification.py new file mode 100644 index 0000000..6f44996 --- /dev/null +++ b/experiments/agent_qualification.py @@ -0,0 +1,1229 @@ +"""Reproducible runner for the agentic matching-engine qualification evaluation. + +The tracked protocol and prompt are public. Candidate snapshots, evaluator gold +manifests, raw transcripts, and workspaces live under ``experiments/private/`` +and are intentionally ignored. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import random +import shutil +import signal +import stat +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Mapping, Sequence + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +PROTOCOL_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-evaluation.md" +PROMPT_PATH = REPOSITORY_ROOT / "experiments" / "agent_qualification_prompt.txt" +PRIVATE_ROOT = REPOSITORY_ROOT / "experiments" / "private" / "agent-qualification-v1" +DEFAULT_MANIFEST_PATH = PRIVATE_ROOT / "cases.json" +DEFAULT_PLAN_PATH = PRIVATE_ROOT / "plan.json" +DEFAULT_RUNS_PATH = PRIVATE_ROOT / "runs" +CODEX_WORK_ROOT = PRIVATE_ROOT / "work" / "codex" +CLAUDE_WORK_ROOT = Path("/private/tmp/tracebook-agent-eval") +PROTOCOL_ID = "agent-qualification-v1" +RANDOMIZATION_SEED = 20260728 +DEFAULT_REPETITIONS = 3 +DEFAULT_TIMEOUT_SECONDS = 120 * 60 + +CODEX_BINARY = Path("/Applications/ChatGPT.app/Contents/Resources/codex") +CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json" +CLAUDE_BINARY = Path("/Users/tazeemmahashin/.local/bin/claude") +DOTNET_BINARY_CANDIDATES = ( + Path("/usr/local/opt/dotnet@8/bin/dotnet"), + Path("/usr/local/bin/dotnet"), +) + +DOCS_TREATMENT = ( + "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public " + "README and conformance documentation. Use it only if it fits the " + "declared contract.\n" +) + +NETWORK_DOMAINS = ( + "pypi.org", + "files.pythonhosted.org", + "registry.npmjs.org", + "crates.io", + "static.crates.io", + "index.crates.io", + "proxy.golang.org", + "sum.golang.org", + "github.com", + "api.github.com", + "codeload.github.com", + "raw.githubusercontent.com", + "*.githubusercontent.com", +) + +SECRET_ENVIRONMENT_NAMES = { + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "OPENAI_API_KEY", + "CODEX_API_KEY", + "CODEX_ACCESS_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "GOOGLE_APPLICATION_CREDENTIALS", + "DATABASE_URL", + "SSH_AUTH_SOCK", +} + + +class EvaluationError(RuntimeError): + """Raised when a frozen evaluation precondition is not satisfied.""" + + +def _utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _snapshot_digest(root: Path) -> str: + """Hash relative names, modes, and bytes without following symlinks.""" + + digest = hashlib.sha256() + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(root).as_posix() + if ".git" in path.relative_to(root).parts: + continue + if path.is_symlink(): + digest.update(b"L\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(os.readlink(path).encode()) + digest.update(b"\0") + elif path.is_file(): + digest.update(b"F\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(f"{path.stat().st_mode & 0o777:o}".encode()) + digest.update(b"\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + digest.update(b"\0") + return digest.hexdigest() + + +def _tree_inventory(root: Path) -> list[dict[str, Any]]: + """Return a stable inventory while rejecting unsafe tree entries.""" + + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise EvaluationError(f"frozen fixture must not contain symlink {relative!r}") + mode = path.stat().st_mode + if stat.S_ISDIR(mode): + continue + if not stat.S_ISREG(mode): + raise EvaluationError(f"frozen fixture must contain only regular files: {relative!r}") + inventory.append( + { + "path": relative, + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + ) + return inventory + + +def _private_path(raw_path: str, description: str) -> Path: + path = (REPOSITORY_ROOT / raw_path).resolve() + try: + path.relative_to(PRIVATE_ROOT.resolve()) + except ValueError as exc: + raise EvaluationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc + return path + + +def _dotnet_layout() -> tuple[Path | None, Path | None, Path | None]: + for configured_binary in DOTNET_BINARY_CANDIDATES: + if not configured_binary.exists(): + continue + launcher_directory = configured_binary.resolve().parent + libexec = launcher_directory.parent / "libexec" + root = libexec if libexec.is_dir() else launcher_directory + return configured_binary, launcher_directory, root + return None, None, None + + +def _load_json(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise EvaluationError(f"cannot read JSON from {path}: {exc}") from exc + if not isinstance(payload, dict): + raise EvaluationError(f"{path} must contain one JSON object") + return payload + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _case_index(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + cases = manifest.get("cases") + if not isinstance(cases, list) or not cases: + raise EvaluationError("case manifest must contain a non-empty cases list") + result: dict[str, dict[str, Any]] = {} + for raw_case in cases: + if not isinstance(raw_case, dict): + raise EvaluationError("every case must be an object") + case_id = raw_case.get("id") + if not isinstance(case_id, str) or not case_id: + raise EvaluationError("every case must have a non-empty id") + if case_id in result: + raise EvaluationError(f"duplicate case id {case_id!r}") + result[case_id] = raw_case + return result + + +def _validate_frozen_tree( + case: Mapping[str, Any], + declaration: Any, + *, + field_name: str, + expected_kind: str, +) -> dict[str, Any]: + if not isinstance(declaration, dict): + raise EvaluationError(f"case {case['id']!r} must declare {field_name!r} as an object") + kind = declaration.get("kind") + if kind == "none": + if set(declaration) != {"kind"}: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} kind 'none' " "must not contain fixture fields" + ) + return {"kind": "none"} + if kind != expected_kind: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} kind must be " f"{expected_kind!r} or 'none'" + ) + + required = ( + "source", + "manifest", + "manifest_sha256", + "tree_sha256", + ) + missing = [name for name in required if not isinstance(declaration.get(name), str)] + if missing: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} is missing string fields: " + ", ".join(missing) + ) + source = _private_path(str(declaration["source"]), f"case {case['id']!r} {field_name!r} source") + fixture_manifest_path = _private_path( + str(declaration["manifest"]), + f"case {case['id']!r} {field_name!r} manifest", + ) + if not source.is_dir(): + raise EvaluationError(f"case {case['id']!r} {field_name!r} source is unavailable: {source}") + if not fixture_manifest_path.is_file(): + raise EvaluationError( + f"case {case['id']!r} {field_name!r} manifest is unavailable: " + f"{fixture_manifest_path}" + ) + actual_manifest_hash = _sha256_file(fixture_manifest_path) + if declaration["manifest_sha256"] != actual_manifest_hash: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} manifest changed: " + f"expected {declaration['manifest_sha256']}, got {actual_manifest_hash}" + ) + + fixture_manifest = _load_json(fixture_manifest_path) + inventory = _tree_inventory(source) + tree_hash = _snapshot_digest(source) + total_bytes = sum(int(item["bytes"]) for item in inventory) + bindings = { + "kind": expected_kind, + "case_id": case["id"], + "snapshot_sha256": case["snapshot_sha256"], + "tree_sha256": tree_hash, + "file_count": len(inventory), + "total_bytes": total_bytes, + "files": inventory, + } + for name, expected in bindings.items(): + if fixture_manifest.get(name) != expected: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} manifest binding " f"{name!r} changed" + ) + if declaration["tree_sha256"] != tree_hash: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} tree changed: " + f"expected {declaration['tree_sha256']}, got {tree_hash}" + ) + return { + "kind": expected_kind, + "source": source, + "manifest": fixture_manifest_path, + "manifest_sha256": actual_manifest_hash, + "tree_sha256": tree_hash, + "file_count": len(inventory), + "total_bytes": total_bytes, + } + + +def validate_manifest(manifest_path: Path) -> dict[str, Any]: + manifest = _load_json(manifest_path) + if manifest.get("protocol") != PROTOCOL_ID: + raise EvaluationError( + f"manifest protocol must be {PROTOCOL_ID!r}, got {manifest.get('protocol')!r}" + ) + expected_protocol_hash = manifest.get("protocol_sha256") + actual_protocol_hash = _sha256_file(PROTOCOL_PATH) + if expected_protocol_hash != actual_protocol_hash: + raise EvaluationError( + "tracked protocol hash changed after case freeze: " + f"expected {expected_protocol_hash}, got {actual_protocol_hash}" + ) + expected_prompt_hash = manifest.get("prompt_sha256") + actual_prompt_hash = _sha256_file(PROMPT_PATH) + if expected_prompt_hash != actual_prompt_hash: + raise EvaluationError( + "tracked prompt hash changed after case freeze: " + f"expected {expected_prompt_hash}, got {actual_prompt_hash}" + ) + + for case in _case_index(manifest).values(): + required = ( + "source", + "snapshot_id", + "revision", + "commands", + "declared_claim", + "excluded_scope", + "snapshot_sha256", + "status", + ) + missing = [field for field in required if field not in case] + if missing: + raise EvaluationError(f"case {case['id']!r} is missing fields: {', '.join(missing)}") + source = (REPOSITORY_ROOT / str(case["source"])).resolve() + if not source.is_dir(): + raise EvaluationError(f"case {case['id']!r} source is not a directory") + if (source / ".git").exists(): + raise EvaluationError(f"case {case['id']!r} source must be origin-stripped") + actual_snapshot_hash = _snapshot_digest(source) + if case["snapshot_sha256"] != actual_snapshot_hash: + raise EvaluationError( + f"case {case['id']!r} snapshot changed: " + f"expected {case['snapshot_sha256']}, got {actual_snapshot_hash}" + ) + if case.get("status") == "ready": + gold_manifest = case.get("gold_manifest") + gold_hash = case.get("gold_sha256") + if not isinstance(gold_manifest, str) or not isinstance(gold_hash, str): + raise EvaluationError( + f"ready case {case['id']!r} must freeze gold_manifest and gold_sha256" + ) + gold_path = (REPOSITORY_ROOT / gold_manifest).resolve() + if not gold_path.is_file(): + raise EvaluationError( + f"case {case['id']!r} gold manifest is unavailable: {gold_path}" + ) + actual_gold_hash = _sha256_file(gold_path) + if gold_hash != actual_gold_hash: + raise EvaluationError( + f"case {case['id']!r} gold manifest changed: " + f"expected {gold_hash}, got {actual_gold_hash}" + ) + _validate_frozen_tree( + case, + case.get("dependency_cache"), + field_name="dependency_cache", + expected_kind="nuget-global-packages-v1", + ) + _validate_frozen_tree( + case, + case.get("native_test_tool"), + field_name="native_test_tool", + expected_kind="xunit-inproc-v1", + ) + return manifest + + +def render_prompt(case: Mapping[str, Any], condition: str) -> str: + prompt = PROMPT_PATH.read_text() + substitutions = { + "case_id": str(case["id"]), + "snapshot_id": str(case["snapshot_id"]), + "revision": str(case["revision"]), + "commands": str(case["commands"]), + "declared_claim": str(case["declared_claim"]), + "excluded_scope": str(case["excluded_scope"]), + } + for name, value in substitutions.items(): + prompt = prompt.replace("{{" + name + "}}", value) + if "{{" in prompt or "}}" in prompt: + raise EvaluationError("unresolved prompt placeholder") + if condition == "baseline": + return prompt + if condition in {"docs", "skill"}: + return prompt.rstrip() + DOCS_TREATMENT + raise EvaluationError(f"unknown treatment condition {condition!r}") + + +def create_plan( + manifest_path: Path, + plan_path: Path, + repetitions: int, + conditions: Sequence[str], +) -> dict[str, Any]: + if repetitions < 1: + raise EvaluationError("repetitions must be positive") + manifest = validate_manifest(manifest_path) + cases = _case_index(manifest) + entries = [ + { + "run_id": f"{condition}__{case_id}__{agent}__r{repetition}", + "condition": condition, + "case_id": case_id, + "agent": agent, + "repetition": repetition, + } + for condition in conditions + for case_id, case in cases.items() + if case.get("status") == "ready" + for agent in ("codex", "claude") + for repetition in range(1, repetitions + 1) + ] + random.Random(RANDOMIZATION_SEED).shuffle(entries) + plan = { + "protocol": PROTOCOL_ID, + "created_at": _utc_now(), + "randomization_seed": RANDOMIZATION_SEED, + "manifest_path": str(manifest_path.resolve()), + "manifest_sha256": _sha256_file(manifest_path), + "runner_sha256": _sha256_file(RUNNER_PATH), + "repetitions": repetitions, + "conditions": list(conditions), + "entries": entries, + } + _write_json(plan_path, plan) + return plan + + +def _clean_agent_environment(scratch: Path) -> dict[str, str]: + secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + secret_prefixes = ("GH_", "GITHUB_", "AWS_", "GOOGLE_") + environment = { + name: value + for name, value in os.environ.items() + if name not in SECRET_ENVIRONMENT_NAMES + and not any(fragment in name.upper() for fragment in secret_fragments) + and not name.upper().startswith(secret_prefixes) + } + + cache_paths = { + "TMPDIR": scratch / "tmp", + "PIP_CACHE_DIR": scratch / "pip-cache", + "UV_CACHE_DIR": scratch / "uv-cache", + "CARGO_HOME": scratch / "cargo-home", + "NUGET_PACKAGES": scratch / "nuget", + "DOTNET_CLI_HOME": scratch / "dotnet-home", + "npm_config_cache": scratch / "npm-cache", + "CODEX_SQLITE_HOME": scratch / "codex-state", + } + for name, path in cache_paths.items(): + path.mkdir(parents=True, exist_ok=True) + environment[name] = str(path) + environment["GIT_TERMINAL_PROMPT"] = "0" + environment["CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR"] = "1" + environment["CLAUDE_CODE_SKIP_PROMPT_HISTORY"] = "1" + rustup_home = Path.home() / ".rustup" + if rustup_home.is_dir(): + environment["RUSTUP_HOME"] = str(rustup_home) + return environment + + +def _copy_frozen_tree(source: Path, target: Path, expected_hash: str) -> None: + if target.exists(): + if not target.is_dir() or any(target.iterdir()): + raise EvaluationError(f"fixture destination must be an empty directory: {target}") + target.rmdir() + try: + subprocess.run( + ("/bin/cp", "-cR", str(source), str(target)), + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError: + shutil.copytree(source, target) + actual_hash = _snapshot_digest(target) + if actual_hash != expected_hash: + raise EvaluationError( + f"fixture copy changed in transit: expected {expected_hash}, got {actual_hash}" + ) + + +def _prepare_case_fixtures(case: Mapping[str, Any], scratch: Path) -> dict[str, dict[str, Any]]: + declarations = ( + ( + "dependency_cache", + "nuget-global-packages-v1", + scratch / "nuget", + ), + ( + "native_test_tool", + "xunit-inproc-v1", + scratch / "native-test-tool", + ), + ) + prepared: dict[str, dict[str, Any]] = {} + for field_name, expected_kind, target in declarations: + fixture = _validate_frozen_tree( + case, + case.get(field_name), + field_name=field_name, + expected_kind=expected_kind, + ) + if fixture["kind"] == "none": + prepared[field_name] = {"kind": "none"} + continue + _copy_frozen_tree( + Path(fixture["source"]), + target, + str(fixture["tree_sha256"]), + ) + prepared[field_name] = { + "kind": expected_kind, + "manifest_sha256": fixture["manifest_sha256"], + "initial_tree_sha256": fixture["tree_sha256"], + "file_count": fixture["file_count"], + "total_bytes": fixture["total_bytes"], + "target": str(target), + } + + shell_home = scratch / "shell-home" + shell_home.mkdir(parents=True, exist_ok=True) + nuget_config = shell_home / ".nuget" / "NuGet" / "NuGet.Config" + nuget_config.parent.mkdir(parents=True, exist_ok=True) + nuget_config.write_text( + '\n' + "\n" + " \n" + " \n" + "\n" + ) + return prepared + + +def _record_final_fixture_state(fixtures: dict[str, dict[str, Any]], scratch: Path) -> None: + targets = { + "dependency_cache": scratch / "nuget", + "native_test_tool": scratch / "native-test-tool", + } + for field_name, fixture in fixtures.items(): + if fixture.get("kind") == "none": + continue + target = targets[field_name] + if not target.is_dir(): + fixture["final_state"] = "missing" + fixture["mutated"] = True + continue + try: + final_hash = _snapshot_digest(target) + except OSError as exc: + fixture["final_state"] = f"unreadable:{type(exc).__name__}" + fixture["mutated"] = True + continue + fixture["final_state"] = "present" + fixture["final_tree_sha256"] = final_hash + fixture["mutated"] = final_hash != fixture["initial_tree_sha256"] + + +def _dotnet_environment(scratch: Path) -> dict[str, str]: + _, _, dotnet_root = _dotnet_layout() + xdg_data_home = scratch / "xdg-data" + migration_sentinel = xdg_data_home / "NuGet" / "Migrations" / "1" + migration_sentinel.parent.mkdir(parents=True, exist_ok=True) + migration_sentinel.touch(exist_ok=True) + values = { + "DOTNET_EnableDiagnostics": "0", + "DOTNET_CLI_TELEMETRY_OPTOUT": "1", + "DOTNET_SKIP_FIRST_TIME_EXPERIENCE": "1", + "DOTNET_CLI_USE_MSBUILD_SERVER": "0", + "MSBUILDUSESERVER": "0", + "NUGET_CERT_REVOCATION_MODE": "offline", + "NuGetAudit": "false", + "HOME": str(scratch / "shell-home"), + # NuGet otherwise creates a named mutex before checking its migration + # state. Claude's macOS sandbox intentionally blocks that global IPC, + # so pre-seed NuGet's documented empty completion marker instead. + "XDG_DATA_HOME": str(xdg_data_home), + } + if dotnet_root is not None: + values["DOTNET_ROOT"] = str(dotnet_root) + return values + + +def _claude_settings(workspace: Path, scratch: Path) -> dict[str, Any]: + user_home = str(Path.home()) + workspace_rule_path = "//" + str(workspace).lstrip("/") + user_home_rule_path = "//" + user_home.lstrip("/") + cargo_bin = Path.home() / ".cargo" / "bin" + rustup = Path.home() / ".rustup" + _, dotnet_launcher_dir, _ = _dotnet_layout() + path_parts = [ + *([str(scratch / "native-test-tool")] if (scratch / "native-test-tool").is_dir() else []), + *([str(dotnet_launcher_dir)] if dotnet_launcher_dir is not None else []), + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + ] + allow_read = [str(workspace)] + if cargo_bin.is_dir(): + path_parts.append(str(cargo_bin)) + allow_read.append(str(cargo_bin)) + if rustup.is_dir(): + allow_read.append(str(rustup)) + for dotnet_read_path in ( + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + ): + if dotnet_read_path.exists(): + allow_read.append(str(dotnet_read_path)) + + environment = { + "PATH": ":".join(path_parts), + "TMPDIR": str(scratch / "tmp"), + "PIP_CACHE_DIR": str(scratch / "pip-cache"), + "UV_CACHE_DIR": str(scratch / "uv-cache"), + "CARGO_HOME": str(scratch / "cargo-home"), + "NUGET_PACKAGES": str(scratch / "nuget"), + "DOTNET_CLI_HOME": str(scratch / "dotnet-home"), + "npm_config_cache": str(scratch / "npm-cache"), + "GIT_TERMINAL_PROMPT": "0", + "CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1", + } + environment.update(_dotnet_environment(scratch)) + + return { + "env": environment, + "permissions": { + "defaultMode": "dontAsk", + # Claude's OS sandbox applies only to Bash and its children. Its + # native file tools need a second, absolute-path allowlist. In + # Claude rules ``//`` (represented by the leading slash added + # above plus the path's own slash) anchors at the filesystem root; + # a single slash would anchor at the settings-file directory. + # Edit rules cover Edit, Write, and NotebookEdit. + "allow": [ + f"Read({workspace_rule_path}/**)", + f"Edit({workspace_rule_path}/**)", + "Bash", + "WebSearch", + "WebFetch", + ], + "deny": [ + f"Read({user_home_rule_path}/**)", + f"Edit({user_home_rule_path}/**)", + f"Edit({workspace_rule_path}/.git/**)", + "Bash(git commit *)", + "Bash(git push *)", + "Bash(gh *)", + "Bash(ssh *)", + "Bash(scp *)", + "Bash(rsync *)", + ], + }, + "sandbox": { + "enabled": True, + "failIfUnavailable": True, + "autoAllowBashIfSandboxed": True, + "allowUnsandboxedCommands": False, + "filesystem": { + "denyRead": [user_home, "/Volumes"], + "allowRead": allow_read, + "denyWrite": [ + user_home, + "/Volumes", + str(workspace / ".git"), + str(scratch / "native-test-tool"), + ], + "allowWrite": [str(workspace), str(scratch)], + }, + "network": {"allowedDomains": list(NETWORK_DOMAINS)}, + }, + } + + +def _codex_permission_profile(workspace: Path, scratch: Path) -> str: + del workspace # Runtime ``:workspace_roots`` supplies the exact workspace. + filesystem = [ + '":root"="deny"', + '":minimal"="read"', + f'{json.dumps(str(scratch))}="write"', + f'{json.dumps(str(scratch / "native-test-tool"))}="read"', + '":workspace_roots"={"."="write",".git"="read"}', + ] + for read_path in ( + Path("/Library/Developer/CommandLineTools"), + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + Path.home() / ".cargo" / "bin", + Path.home() / ".rustup", + ): + if read_path.exists(): + filesystem.append(f'{json.dumps(str(read_path))}="read"') + domains = ",".join(f'{json.dumps(domain)}="allow"' for domain in NETWORK_DOMAINS) + return ( + "permissions.agent-eval={" + f"filesystem={{{','.join(filesystem)}}}," + 'network={enabled=true,mode="limited",allow_upstream_proxy=false,' + f"domains={{{domains}}}}}" + "}" + ) + + +def _codex_shell_environment(scratch: Path) -> dict[str, str]: + _, dotnet_launcher_dir, _ = _dotnet_layout() + path_parts = [ + *([str(scratch / "native-test-tool")] if (scratch / "native-test-tool").is_dir() else []), + *([str(dotnet_launcher_dir)] if dotnet_launcher_dir is not None else []), + "/Library/Developer/CommandLineTools/usr/bin", + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + ] + cargo_bin = Path.home() / ".cargo" / "bin" + if cargo_bin.is_dir(): + path_parts.append(str(cargo_bin)) + values = { + "PATH": ":".join(path_parts), + "TMPDIR": str(scratch / "tmp"), + "PIP_CACHE_DIR": str(scratch / "pip-cache"), + "UV_CACHE_DIR": str(scratch / "uv-cache"), + "CARGO_HOME": str(scratch / "cargo-home"), + "NUGET_PACKAGES": str(scratch / "nuget"), + "DOTNET_CLI_HOME": str(scratch / "dotnet-home"), + "npm_config_cache": str(scratch / "npm-cache"), + "GIT_TERMINAL_PROMPT": "0", + "GIT_OPTIONAL_LOCKS": "0", + } + rustup_home = Path.home() / ".rustup" + if rustup_home.is_dir(): + values["RUSTUP_HOME"] = str(rustup_home) + values.update(_dotnet_environment(scratch)) + return values + + +def _codex_command(workspace: Path, scratch: Path) -> list[str]: + permission_profile = _codex_permission_profile(workspace, scratch) + shell_environment = ",".join( + f"{json.dumps(name)}={json.dumps(value)}" + for name, value in _codex_shell_environment(scratch).items() + ) + secret_excludes = ( + 'shell_environment_policy.exclude=["*KEY*","*TOKEN*","*SECRET*",' + '"*PASSWORD*","*CREDENTIAL*","GH_*","GITHUB_*","AWS_*","GOOGLE_*",' + '"DATABASE_URL","CODEX_HOME","SSH_AUTH_SOCK"]' + ) + return [ + str(CODEX_BINARY), + "--ask-for-approval", + "never", + "--search", + "--disable", + "apps", + "--disable", + "plugins", + "--disable", + "remote_plugin", + "--disable", + "skill_search", + "--disable", + "tool_suggest", + "--disable", + "plugin_sharing", + "--disable", + "workspace_dependencies", + "exec", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + "--strict-config", + "--skip-git-repo-check", + "--json", + "--color", + "never", + "--model", + "gpt-5.6-sol", + "--cd", + str(workspace), + "-c", + "features.network_proxy.enabled=true", + "-c", + 'default_permissions="agent-eval"', + "-c", + permission_profile, + "-c", + "project_doc_max_bytes=0", + "-c", + "project_doc_fallback_filenames=[]", + "-c", + 'model_reasoning_effort="high"', + "-c", + "allow_login_shell=false", + "-c", + 'history.persistence="none"', + "-c", + 'shell_environment_policy.inherit="core"', + "-c", + f"shell_environment_policy.set={{{shell_environment}}}", + "-c", + secret_excludes, + "-c", + "features.apps=false", + "-c", + "features.remote_plugin=false", + "-c", + "features.multi_agent=false", + "-c", + "features.memories=false", + "-c", + "features.hooks=false", + "-c", + "features.skill_mcp_dependency_install=false", + "-c", + "features.goals=false", + "-c", + "agents.enabled=false", + "-c", + "check_for_update_on_startup=false", + "-c", + "feedback.enabled=false", + "-c", + "analytics.enabled=false", + "-c", + 'personality="none"', + "-", + ] + + +def _prepare_codex_environment( + external_run_root: Path, + scratch: Path, + environment: dict[str, str], +) -> Path: + if not CODEX_AUTH_PATH.is_file(): + raise EvaluationError(f"Codex auth is unavailable at {CODEX_AUTH_PATH}") + codex_home = external_run_root / "codex-home" + codex_home.mkdir() + codex_home.chmod(0o700) + codex_state = codex_home / "state" + codex_state.mkdir() + (codex_home / "auth.json").symlink_to(CODEX_AUTH_PATH) + shell_home = scratch / "shell-home" + shell_home.mkdir(exist_ok=True) + environment.update(_codex_shell_environment(scratch)) + environment["CODEX_HOME"] = str(codex_home) + environment["CODEX_SQLITE_HOME"] = str(codex_state) + return codex_home + + +def _claude_command(settings_path: Path) -> list[str]: + return [ + str(CLAUDE_BINARY), + "-p", + "--safe-mode", + "--no-session-persistence", + "--no-chrome", + "--disable-slash-commands", + "--strict-mcp-config", + "--mcp-config", + '{"mcpServers":{}}', + "--settings", + str(settings_path), + "--permission-mode", + "dontAsk", + "--tools", + "Read,Glob,Grep,Edit,Write,Bash,WebSearch,WebFetch", + "--model", + "opus", + "--effort", + "high", + "--max-budget-usd", + "50", + "--output-format", + "stream-json", + "--verbose", + ] + + +def _run_subprocess( + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + prompt_stdin: str | None, + stdout_path: Path, + stderr_path: Path, + timeout_seconds: int, +) -> tuple[int, bool, float]: + started = time.monotonic() + timed_out = False + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + process = subprocess.Popen( + list(command), + cwd=cwd, + env=dict(environment), + stdin=subprocess.PIPE if prompt_stdin is not None else subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) + try: + process.communicate( + prompt_stdin.encode() if prompt_stdin is not None else None, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + elapsed = time.monotonic() - started + return process.returncode, timed_out, elapsed + + +def _run_checked(command: Sequence[str], cwd: Path) -> str: + completed = subprocess.run( + list(command), + cwd=cwd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + return completed.stdout + + +def _binary_version(binary: Path, environment: Mapping[str, str]) -> str: + completed = subprocess.run( + (str(binary), "--version"), + env=dict(environment), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=15, + ) + return completed.stdout.strip() + + +def _reported_model(transcript_path: Path, agent: str) -> tuple[str, str]: + if agent == "codex": + # Codex's JSONL protocol does not currently repeat the selected model; + # the CLI's exact, non-alias model argument is the authoritative value. + return "gpt-5.6-sol", "command" + with transcript_path.open() as handle: + for line in handle: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + model = event.get("model") + if isinstance(model, str) and model: + return model, "transcript" + message = event.get("message") + if isinstance(message, dict): + model = message.get("model") + if isinstance(model, str) and model: + return model, "transcript" + return "opus", "command-fallback" + + +def _initialize_snapshot(source: Path, workspace: Path) -> None: + shutil.copytree(source, workspace, symlinks=True) + _run_checked(("git", "init", "-q"), workspace) + _run_checked(("git", "config", "user.name", "Tracebook evaluator"), workspace) + _run_checked(("git", "config", "user.email", "evaluator@tracebook.invalid"), workspace) + _run_checked(("git", "config", "commit.gpgsign", "false"), workspace) + _run_checked(("git", "add", "-A"), workspace) + _run_checked(("git", "commit", "-q", "-m", "Frozen candidate snapshot"), workspace) + + +def _write_workspace_evidence(workspace: Path, run_root: Path) -> None: + (run_root / "git-status.txt").write_text(_run_checked(("git", "status", "--short"), workspace)) + (run_root / "workspace.patch").write_text( + _run_checked(("git", "diff", "--binary", "HEAD"), workspace) + ) + inventory: list[dict[str, Any]] = [] + excluded_parts = {".git", "bin", "obj", "target", "node_modules"} + for path in sorted(workspace.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(workspace) + if any(part in excluded_parts for part in relative.parts): + continue + if path.is_file() and not path.is_symlink(): + inventory.append( + { + "path": relative.as_posix(), + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + ) + _write_json(run_root / "workspace-files.json", {"files": inventory}) + + +def execute_run( + manifest_path: Path, + *, + case_id: str, + agent: str, + repetition: int, + condition: str, + runs_path: Path, + timeout_seconds: int, +) -> dict[str, Any]: + manifest = validate_manifest(manifest_path) + cases = _case_index(manifest) + try: + case = cases[case_id] + except KeyError as exc: + raise EvaluationError(f"unknown case {case_id!r}") from exc + if case.get("status") != "ready": + raise EvaluationError(f"case {case_id!r} is not ready") + if agent not in {"codex", "claude"}: + raise EvaluationError(f"unknown agent {agent!r}") + if repetition < 1: + raise EvaluationError("repetition must be positive") + if condition == "skill": + raise EvaluationError("skill-assisted runs remain locked until baseline grading") + + run_id = f"{condition}__{case_id}__{agent}__r{repetition}" + run_root = runs_path.resolve() / run_id + if run_root.exists(): + raise EvaluationError(f"refusing to overwrite existing run {run_root}") + + source = (REPOSITORY_ROOT / str(case["source"])).resolve() + work_root = CODEX_WORK_ROOT if agent == "codex" else CLAUDE_WORK_ROOT + external_run_root = work_root / run_id + if external_run_root.exists(): + raise EvaluationError(f"refusing to reuse external workspace {external_run_root}") + workspace = external_run_root / "workspace" + scratch = external_run_root / "scratch" + transcript_path = run_root / f"{agent}.jsonl" + stderr_path = run_root / f"{agent}.stderr" + run_root.mkdir(parents=True) + scratch.mkdir(parents=True) + _initialize_snapshot(source, workspace) + + prompt = render_prompt(case, condition) + (run_root / "prompt.txt").write_text(prompt) + environment = _clean_agent_environment(scratch) + fixtures = _prepare_case_fixtures(case, scratch) + settings_path: Path | None = None + if agent == "codex": + if not CODEX_BINARY.is_file(): + raise EvaluationError(f"Codex binary is unavailable at {CODEX_BINARY}") + _prepare_codex_environment(external_run_root, scratch, environment) + command = _codex_command(workspace, scratch) + prompt_stdin = prompt + else: + if not CLAUDE_BINARY.is_file(): + raise EvaluationError(f"Claude binary is unavailable at {CLAUDE_BINARY}") + settings_path = run_root / "claude-settings.json" + _write_json(settings_path, _claude_settings(workspace, scratch)) + command = _claude_command(settings_path) + prompt_stdin = prompt + + cli_version = _binary_version(CODEX_BINARY if agent == "codex" else CLAUDE_BINARY, environment) + started_at = _utc_now() + metadata: dict[str, Any] = { + "protocol": PROTOCOL_ID, + "run_id": run_id, + "condition": condition, + "case_id": case_id, + "agent": agent, + "repetition": repetition, + "started_at": started_at, + "timeout_seconds": timeout_seconds, + "manifest_sha256": _sha256_file(manifest_path), + "gold_manifest_sha256": str(case["gold_sha256"]), + "runner_sha256": _sha256_file(RUNNER_PATH), + "protocol_sha256": _sha256_file(PROTOCOL_PATH), + "prompt_template_sha256": _sha256_file(PROMPT_PATH), + "rendered_prompt_sha256": _sha256_bytes(prompt.encode()), + "snapshot_sha256": _snapshot_digest(source), + "command": command, + "prompt_transport": "stdin", + "cli_version": cli_version, + "requested_model": "gpt-5.6-sol" if agent == "codex" else "opus", + "removed_environment_names": sorted(SECRET_ENVIRONMENT_NAMES), + "fixtures": fixtures, + } + _write_json(run_root / "metadata.json", metadata) + + exit_code, timed_out, elapsed_seconds = _run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt_stdin, + stdout_path=transcript_path, + stderr_path=stderr_path, + timeout_seconds=timeout_seconds, + ) + _write_workspace_evidence(workspace, run_root) + _record_final_fixture_state(fixtures, scratch) + shutil.copytree( + workspace, + run_root / "workspace-final", + symlinks=True, + ignore=shutil.ignore_patterns( + ".git", "bin", "obj", "target", "node_modules", ".eval-cache" + ), + ) + + metadata.update( + { + "completed_at": _utc_now(), + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed_seconds, + "transcript_sha256": _sha256_file(transcript_path), + "stderr_sha256": _sha256_file(stderr_path), + "fixtures": fixtures, + } + ) + model_identifier, model_identifier_source = _reported_model(transcript_path, agent) + metadata["model_identifier"] = model_identifier + metadata["model_identifier_source"] = model_identifier_source + _write_json(run_root / "metadata.json", metadata) + shutil.rmtree(external_run_root) + return metadata + + +def _parse_conditions(raw: str) -> list[str]: + conditions = [item.strip() for item in raw.split(",") if item.strip()] + unknown = [item for item in conditions if item not in {"baseline", "docs", "skill"}] + if unknown: + raise EvaluationError(f"unknown conditions: {', '.join(unknown)}") + if not conditions: + raise EvaluationError("at least one condition is required") + return conditions + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser("validate", help="validate frozen inputs") + validate.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH) + + plan = subparsers.add_parser("plan", help="create a deterministic run order") + plan.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH) + plan.add_argument("--output", type=Path, default=DEFAULT_PLAN_PATH) + plan.add_argument("--repetitions", type=int, default=DEFAULT_REPETITIONS) + plan.add_argument("--conditions", default="baseline") + + run = subparsers.add_parser("run", help="execute one isolated run") + run.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH) + run.add_argument("--case", required=True) + run.add_argument("--agent", choices=("codex", "claude"), required=True) + run.add_argument("--repetition", type=int, required=True) + run.add_argument("--condition", choices=("baseline", "docs", "skill"), default="baseline") + run.add_argument("--runs-path", type=Path, default=DEFAULT_RUNS_PATH) + run.add_argument("--timeout-seconds", type=int, default=DEFAULT_TIMEOUT_SECONDS) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + arguments = parser.parse_args(argv) + try: + if arguments.command == "validate": + manifest = validate_manifest(arguments.manifest) + print( + json.dumps( + { + "protocol": manifest["protocol"], + "cases": sorted(_case_index(manifest)), + "status": "valid", + }, + sort_keys=True, + ) + ) + elif arguments.command == "plan": + plan = create_plan( + arguments.manifest, + arguments.output, + arguments.repetitions, + _parse_conditions(arguments.conditions), + ) + print( + json.dumps( + { + "entries": len(plan["entries"]), + "output": str(arguments.output), + "status": "created", + }, + sort_keys=True, + ) + ) + elif arguments.command == "run": + metadata = execute_run( + arguments.manifest, + case_id=arguments.case, + agent=arguments.agent, + repetition=arguments.repetition, + condition=arguments.condition, + runs_path=arguments.runs_path, + timeout_seconds=arguments.timeout_seconds, + ) + print(json.dumps(metadata, sort_keys=True)) + else: # pragma: no cover - argparse enforces the command set + parser.error(f"unsupported command {arguments.command!r}") + except (EvaluationError, OSError, subprocess.CalledProcessError) as exc: + print(f"agent qualification evaluation failed: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/agent_qualification_generalization.py b/experiments/agent_qualification_generalization.py new file mode 100644 index 0000000..9390453 --- /dev/null +++ b/experiments/agent_qualification_generalization.py @@ -0,0 +1,541 @@ +"""Freeze and validate the v2 held-out agent-qualification cohort. + +This module intentionally does not launch provider CLIs. Execution remains +blocked until the user explicitly authorizes transmission of the new public +candidate snapshots and frozen treatment material to each provider. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import stat +from pathlib import Path +from typing import Any, Mapping, Sequence + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +PROTOCOL_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-generalization-v2.md" +PROMPT_PATH = REPOSITORY_ROOT / "experiments" / "agent_qualification_generalization_prompt.txt" +PRIVATE_ROOT = REPOSITORY_ROOT / "experiments" / "private" / "agent-qualification-v2" +DRAFT_PATH = PRIVATE_ROOT / "draft.json" +MANIFEST_PATH = PRIVATE_ROOT / "cases.json" +PLAN_PATH = PRIVATE_ROOT / "plan.json" +FREEZE_PATH = PRIVATE_ROOT / "freeze.json" +MANIFESTS_ROOT = PRIVATE_ROOT / "manifests" +SKILL_PATH = ( + PRIVATE_ROOT + / "treatments" + / "tracebook-qualify-matching-engine" + / "SKILL.md" +) + +PROTOCOL_ID = "agent-qualification-generalization-v2" +RANDOMIZATION_SEED = 20260730 +REPETITIONS = 3 +AGENTS = ("codex", "claude") +CONDITIONS = ("docs", "skill") + +DOCS_TREATMENT = ( + "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public " + "README and conformance documentation. Use it only if it fits the " + "declared contract.\n" +) + + +class GeneralizationError(RuntimeError): + """Raised when a frozen v2 precondition is not satisfied.""" + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _snapshot_digest(root: Path) -> str: + """Hash relative file names, modes, links, and bytes deterministically.""" + + digest = hashlib.sha256() + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative_path = path.relative_to(root) + if ".git" in relative_path.parts: + continue + relative = relative_path.as_posix() + if path.is_symlink(): + digest.update(b"L\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(os.readlink(path).encode()) + digest.update(b"\0") + elif path.is_file(): + digest.update(b"F\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(f"{path.stat().st_mode & 0o777:o}".encode()) + digest.update(b"\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + digest.update(b"\0") + return digest.hexdigest() + + +def _tree_inventory(root: Path) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise GeneralizationError(f"frozen tree must not contain symlink {relative!r}") + mode = path.stat().st_mode + if stat.S_ISDIR(mode): + continue + if not stat.S_ISREG(mode): + raise GeneralizationError(f"frozen tree contains unsafe entry {relative!r}") + inventory.append( + { + "path": relative, + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + ) + return inventory + + +def _load_json(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise GeneralizationError(f"cannot read JSON from {path}: {exc}") from exc + if not isinstance(payload, dict): + raise GeneralizationError(f"{path} must contain one JSON object") + return payload + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _relative(path: Path) -> str: + return path.resolve().relative_to(REPOSITORY_ROOT.resolve()).as_posix() + + +def _private_path(raw_path: str, description: str) -> Path: + path = (REPOSITORY_ROOT / raw_path).resolve() + try: + path.relative_to(PRIVATE_ROOT.resolve()) + except ValueError as exc: + raise GeneralizationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc + return path + + +def _cases(payload: Mapping[str, Any]) -> list[dict[str, Any]]: + raw_cases = payload.get("cases") + if not isinstance(raw_cases, list) or not raw_cases: + raise GeneralizationError("case payload must contain a non-empty cases list") + result: list[dict[str, Any]] = [] + seen: set[str] = set() + for raw in raw_cases: + if not isinstance(raw, dict): + raise GeneralizationError("every case must be an object") + case_id = raw.get("id") + if not isinstance(case_id, str) or not case_id: + raise GeneralizationError("every case must have a non-empty id") + if case_id in seen: + raise GeneralizationError(f"duplicate case id {case_id!r}") + seen.add(case_id) + result.append(dict(raw)) + return result + + +def _freeze_tree( + *, + case_id: str, + name: str, + source_path: str, + snapshot_sha256: str, +) -> dict[str, Any]: + source = _private_path(source_path, f"{case_id} {name} source") + if not source.is_dir(): + raise GeneralizationError(f"{case_id} {name} source is unavailable: {source}") + inventory = _tree_inventory(source) + tree_sha256 = _snapshot_digest(source) + manifest = { + "kind": "frozen-tree-v1", + "case_id": case_id, + "name": name, + "snapshot_sha256": snapshot_sha256, + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": sum(int(item["bytes"]) for item in inventory), + "files": inventory, + } + manifest_path = MANIFESTS_ROOT / f"{case_id}--{name}.json" + _write_json(manifest_path, manifest) + return { + "kind": "frozen-tree-v1", + "source": _relative(source), + "manifest": _relative(manifest_path), + "manifest_sha256": _sha256_file(manifest_path), + "tree_sha256": tree_sha256, + } + + +def _validate_tree( + case: Mapping[str, Any], + declaration: Any, + *, + name: str, +) -> dict[str, Any]: + if not isinstance(declaration, dict) or declaration.get("kind") != "frozen-tree-v1": + raise GeneralizationError(f"case {case['id']!r} {name!r} must be frozen-tree-v1") + for field in ("source", "manifest", "manifest_sha256", "tree_sha256"): + if not isinstance(declaration.get(field), str): + raise GeneralizationError(f"case {case['id']!r} {name!r} lacks {field!r}") + source = _private_path(str(declaration["source"]), f"{case['id']} {name} source") + manifest_path = _private_path( + str(declaration["manifest"]), f"{case['id']} {name} manifest" + ) + if not source.is_dir() or not manifest_path.is_file(): + raise GeneralizationError(f"case {case['id']!r} {name!r} tree is unavailable") + manifest_hash = _sha256_file(manifest_path) + if manifest_hash != declaration["manifest_sha256"]: + raise GeneralizationError(f"case {case['id']!r} {name!r} manifest drifted") + manifest = _load_json(manifest_path) + inventory = _tree_inventory(source) + tree_sha256 = _snapshot_digest(source) + expected = { + "kind": "frozen-tree-v1", + "case_id": case["id"], + "name": name, + "snapshot_sha256": case["snapshot_sha256"], + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": sum(int(item["bytes"]) for item in inventory), + "files": inventory, + } + if manifest != expected: + raise GeneralizationError(f"case {case['id']!r} {name!r} inventory drifted") + if declaration["tree_sha256"] != tree_sha256: + raise GeneralizationError(f"case {case['id']!r} {name!r} tree drifted") + return { + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": expected["total_bytes"], + } + + +def freeze() -> dict[str, Any]: + """Create the immutable v2 case manifest, plan, and freeze binding.""" + + existing = [path for path in (MANIFEST_PATH, PLAN_PATH, FREEZE_PATH) if path.exists()] + if existing: + raise GeneralizationError( + "refusing to overwrite frozen v2 files: " + ", ".join(str(path) for path in existing) + ) + draft = _load_json(DRAFT_PATH) + if draft.get("protocol") != PROTOCOL_ID: + raise GeneralizationError(f"draft protocol must be {PROTOCOL_ID!r}") + if not PROTOCOL_PATH.is_file() or not PROMPT_PATH.is_file() or not SKILL_PATH.is_file(): + raise GeneralizationError("tracked protocol, prompt, and v2 skill must exist before freeze") + + frozen_cases: list[dict[str, Any]] = [] + for raw_case in _cases(draft): + required = ( + "source", + "revision", + "commands", + "declared_claim", + "excluded_scope", + "origin", + "gold_manifest", + "dependency_source", + "gold_evidence", + "status", + ) + missing = [field for field in required if field not in raw_case] + if missing: + raise GeneralizationError( + f"case {raw_case['id']!r} is missing fields: {', '.join(missing)}" + ) + source = _private_path(str(raw_case["source"]), f"{raw_case['id']} source") + if not source.is_dir() or (source / ".git").exists(): + raise GeneralizationError( + f"case {raw_case['id']!r} source must be an origin-stripped directory" + ) + snapshot_sha256 = _snapshot_digest(source) + gold_path = _private_path( + str(raw_case["gold_manifest"]), f"{raw_case['id']} gold manifest" + ) + if not gold_path.is_file(): + raise GeneralizationError(f"case {raw_case['id']!r} gold is unavailable") + evidence_specs = raw_case["gold_evidence"] + if not isinstance(evidence_specs, list): + raise GeneralizationError(f"case {raw_case['id']!r} gold_evidence must be a list") + evidence: list[dict[str, Any]] = [] + for item in evidence_specs: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise GeneralizationError("gold evidence entries require name and source") + evidence.append( + { + "name": item["name"], + **_freeze_tree( + case_id=str(raw_case["id"]), + name=str(item["name"]), + source_path=str(item.get("source", "")), + snapshot_sha256=snapshot_sha256, + ), + } + ) + dependency = _freeze_tree( + case_id=str(raw_case["id"]), + name="dependency-cache", + source_path=str(raw_case["dependency_source"]), + snapshot_sha256=snapshot_sha256, + ) + frozen_cases.append( + { + "id": raw_case["id"], + "source": _relative(source), + "snapshot_id": f"sha256:{snapshot_sha256}", + "snapshot_sha256": snapshot_sha256, + "revision": raw_case["revision"], + "commands": raw_case["commands"], + "declared_claim": raw_case["declared_claim"], + "excluded_scope": raw_case["excluded_scope"], + "origin": raw_case["origin"], + "gold_manifest": _relative(gold_path), + "gold_sha256": _sha256_file(gold_path), + "dependency_cache": dependency, + "gold_evidence": evidence, + "status": raw_case["status"], + } + ) + + manifest = { + "protocol": PROTOCOL_ID, + "protocol_sha256": _sha256_file(PROTOCOL_PATH), + "prompt_sha256": _sha256_file(PROMPT_PATH), + "skill_sha256": _sha256_file(SKILL_PATH), + "authorization": { + "codex": "pending-explicit-case-specific-authorization", + "claude": "pending-explicit-case-specific-authorization", + }, + "cases": frozen_cases, + } + _write_json(MANIFEST_PATH, manifest) + + plan = create_plan(manifest) + _write_json(PLAN_PATH, plan) + freeze_payload = { + "protocol": PROTOCOL_ID, + "randomization_seed": RANDOMIZATION_SEED, + "repetitions": REPETITIONS, + "agents": list(AGENTS), + "conditions": list(CONDITIONS), + "bindings": { + "draft": {"path": _relative(DRAFT_PATH), "sha256": _sha256_file(DRAFT_PATH)}, + "manifest": { + "path": _relative(MANIFEST_PATH), + "sha256": _sha256_file(MANIFEST_PATH), + }, + "plan": {"path": _relative(PLAN_PATH), "sha256": _sha256_file(PLAN_PATH)}, + "protocol": { + "path": _relative(PROTOCOL_PATH), + "sha256": _sha256_file(PROTOCOL_PATH), + }, + "prompt": {"path": _relative(PROMPT_PATH), "sha256": _sha256_file(PROMPT_PATH)}, + "runner": {"path": _relative(RUNNER_PATH), "sha256": _sha256_file(RUNNER_PATH)}, + "skill": {"path": _relative(SKILL_PATH), "sha256": _sha256_file(SKILL_PATH)}, + }, + "execution_authorized": False, + "execution_blocker": ( + "Explicit provider authorization for the new C4/C5 public snapshots " + "and frozen treatment material has not been recorded." + ), + } + _write_json(FREEZE_PATH, freeze_payload) + validate() + return freeze_payload + + +def validate_manifest() -> dict[str, Any]: + manifest = _load_json(MANIFEST_PATH) + if manifest.get("protocol") != PROTOCOL_ID: + raise GeneralizationError(f"manifest protocol must be {PROTOCOL_ID!r}") + tracked = { + "protocol_sha256": PROTOCOL_PATH, + "prompt_sha256": PROMPT_PATH, + "skill_sha256": SKILL_PATH, + } + for field, path in tracked.items(): + if manifest.get(field) != _sha256_file(path): + raise GeneralizationError(f"{field} drifted after freeze") + for case in _cases(manifest): + for field in ( + "source", + "snapshot_id", + "snapshot_sha256", + "revision", + "commands", + "declared_claim", + "excluded_scope", + "origin", + "gold_manifest", + "gold_sha256", + "dependency_cache", + "gold_evidence", + "status", + ): + if field not in case: + raise GeneralizationError(f"case {case['id']!r} is missing {field!r}") + source = _private_path(str(case["source"]), f"{case['id']} source") + if not source.is_dir() or (source / ".git").exists(): + raise GeneralizationError(f"case {case['id']!r} source is unavailable or not stripped") + snapshot_sha256 = _snapshot_digest(source) + if snapshot_sha256 != case["snapshot_sha256"]: + raise GeneralizationError(f"case {case['id']!r} snapshot drifted") + if case["snapshot_id"] != f"sha256:{snapshot_sha256}": + raise GeneralizationError(f"case {case['id']!r} snapshot_id is inconsistent") + gold_path = _private_path(str(case["gold_manifest"]), f"{case['id']} gold") + if _sha256_file(gold_path) != case["gold_sha256"]: + raise GeneralizationError(f"case {case['id']!r} gold drifted") + _validate_tree(case, case["dependency_cache"], name="dependency-cache") + evidence = case["gold_evidence"] + if not isinstance(evidence, list): + raise GeneralizationError(f"case {case['id']!r} gold_evidence must be a list") + names: set[str] = set() + for item in evidence: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise GeneralizationError("gold evidence entries require a name") + if item["name"] in names: + raise GeneralizationError(f"duplicate gold evidence name {item['name']!r}") + names.add(item["name"]) + _validate_tree(case, item, name=item["name"]) + return manifest + + +def create_plan(manifest: Mapping[str, Any]) -> dict[str, Any]: + ready = [case for case in _cases(manifest) if case.get("status") == "ready"] + entries: list[dict[str, Any]] = [] + for case in ready: + for agent in AGENTS: + for condition in CONDITIONS: + for repetition in range(1, REPETITIONS + 1): + entries.append( + { + "id": f"{condition}__{case['id']}__{agent}__r{repetition}", + "case_id": case["id"], + "agent": agent, + "condition": condition, + "repetition": repetition, + } + ) + random.Random(RANDOMIZATION_SEED).shuffle(entries) + for index, entry in enumerate(entries, 1): + entry["order"] = index + return { + "protocol": PROTOCOL_ID, + "randomization_seed": RANDOMIZATION_SEED, + "repetitions": REPETITIONS, + "agents": list(AGENTS), + "conditions": list(CONDITIONS), + "entry_count": len(entries), + "entries": entries, + } + + +def validate_plan(manifest: Mapping[str, Any]) -> dict[str, Any]: + plan = _load_json(PLAN_PATH) + expected = create_plan(manifest) + if plan != expected: + raise GeneralizationError("seeded v2 plan drifted") + return plan + + +def validate() -> dict[str, Any]: + manifest = validate_manifest() + plan = validate_plan(manifest) + freeze_payload = _load_json(FREEZE_PATH) + if freeze_payload.get("protocol") != PROTOCOL_ID: + raise GeneralizationError(f"freeze protocol must be {PROTOCOL_ID!r}") + bindings = freeze_payload.get("bindings") + if not isinstance(bindings, dict): + raise GeneralizationError("freeze bindings must be an object") + expected_paths = { + "draft": DRAFT_PATH, + "manifest": MANIFEST_PATH, + "plan": PLAN_PATH, + "protocol": PROTOCOL_PATH, + "prompt": PROMPT_PATH, + "runner": RUNNER_PATH, + "skill": SKILL_PATH, + } + for name, path in expected_paths.items(): + binding = bindings.get(name) + if not isinstance(binding, dict): + raise GeneralizationError(f"freeze lacks {name!r} binding") + if binding.get("path") != _relative(path) or binding.get("sha256") != _sha256_file(path): + raise GeneralizationError(f"freeze {name!r} binding drifted") + if freeze_payload.get("execution_authorized") is not False: + raise GeneralizationError("frozen cohort must remain execution-blocked") + return { + "protocol": PROTOCOL_ID, + "cases": len(_cases(manifest)), + "runs": plan["entry_count"], + "execution_authorized": False, + } + + +def render_prompt(case: Mapping[str, Any], condition: str) -> str: + if condition not in CONDITIONS: + raise GeneralizationError(f"condition must be one of {CONDITIONS!r}") + prompt = PROMPT_PATH.read_text() + substitutions = { + "case_id": case["id"], + "snapshot_id": case["snapshot_id"], + "revision": case["revision"], + "commands": case["commands"], + "declared_claim": case["declared_claim"], + "excluded_scope": case["excluded_scope"], + } + for name, value in substitutions.items(): + prompt = prompt.replace("{{" + name + "}}", str(value)) + if "{{" in prompt or "}}" in prompt: + raise GeneralizationError("unresolved prompt placeholder") + return prompt + DOCS_TREATMENT + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("freeze", "validate", "summary")) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "freeze": + payload = freeze() + elif args.command == "validate": + payload = validate() + else: + payload = validate() + payload["authorization_blocker"] = _load_json(FREEZE_PATH).get( + "execution_blocker" + ) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + except GeneralizationError as exc: + print(f"Generalization experiment error: {exc}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/agent_qualification_generalization_prompt.txt b/experiments/agent_qualification_generalization_prompt.txt new file mode 100644 index 0000000..c0f6b44 --- /dev/null +++ b/experiments/agent_qualification_generalization_prompt.txt @@ -0,0 +1,53 @@ +You are the release-validation engineer for a small venue-infrastructure team +that owns the matching logic in this repository. + +CASE: {{case_id}} +REPOSITORY SNAPSHOT: {{snapshot_id}} +PINNED REVISION: {{revision}} +SUPPORTED BUILD/TEST COMMANDS: {{commands}} +OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}} +EXCLUDED SCOPE: {{excluded_scope}} + +Determine whether this pinned repository can produce safe release evidence for +the declared matching semantics. Do not optimize for finding a bug. A correct +result may be conformant behavior, a candidate defect, an adapter/protocol +defect, a profile mismatch, insufficient observability, or an ambiguity that +requires maintainer adjudication. + +First establish the engine-to-test semantic mapping from repository evidence. +Do not silently invent lifecycle behavior, queue order, numeric conversions, +identities, matching responsibility, or unsupported features. Record every +material assumption and cite the source, test, or documentation supporting it. + +You may search for public development tools and their documentation, but do not +search for this candidate repository, revision, issues, known failures, prior +adapters, or experiment answers. Do not change production matching behavior. +Evaluation-only adapters, probes, artifacts, and native tests are allowed. + +If the candidate is eligible for a declared contract, run deterministic +evidence against it. If a disagreement appears, first rule out adapter, +protocol, normalization, observation, and profile errors. Attribute a +candidate defect only when intended behavior is independently established. + +Every terminal class requires two mechanically fresh evidence runs. Each run +must start from a fresh candidate copy, fresh build/cache target, fresh process, +and absent-before-use output path. Preserve both run identities and compare +their deterministic results. Do not overwrite an incomplete bundle or call a +second command over retained state a clean reproduction. + +Deliver: + +1. `CLASSIFICATION.md`: terminal class, scope, evidence, first disagreement + when applicable, and remaining uncertainty. +2. `MAPPING.md`: matching responsibility, event, numeric, identity, trade, + rejection, and ordered-state mappings with citations. +3. `COMMANDS.log`: every material command, working directory, target, and exit + code. +4. Applicable unmodified machine-readable qualification/failure artifacts. +5. `REPRODUCTION.md`: both fresh run identities, preconditions, outputs, and + equality checks. +6. A native regression only when candidate-defect intent is established. +7. A concise maintainer handoff that makes no claim beyond the evidence. + +Do not commit, push, open an issue, or contact a maintainer. Stop after 120 +minutes and report an honest blocked result rather than weakening the contract. diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py new file mode 100644 index 0000000..48da387 --- /dev/null +++ b/experiments/agent_qualification_generalization_runs.py @@ -0,0 +1,1027 @@ +"""Execute the separately frozen v2 agent-qualification run matrix. + +The primary v2 cohort is immutable. This companion freezes machine/provider +bindings, verifies native skill delivery on synthetic material, enforces +provider-specific authorization, and records technical evidence without +grading semantic outcomes. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import socket +import subprocess +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +if __package__: + from . import agent_qualification as helpers + from . import agent_qualification_generalization as cohort + from . import agent_qualification_treatments as delivery +else: # Support direct execution from the repository root. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import agent_qualification as helpers # type: ignore[no-redef] + import agent_qualification_generalization as cohort # type: ignore[no-redef] + import agent_qualification_treatments as delivery # type: ignore[no-redef] + +ExecutionError = helpers.EvaluationError + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +TEST_PATH = REPOSITORY_ROOT / "tests" / "test_agent_qualification_generalization_runs.py" +ADDENDUM_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-generalization-execution-v2.md" +PRIVATE_ROOT = cohort.PRIVATE_ROOT +EXECUTION_ROOT = PRIVATE_ROOT / "execution" +RUNS_ROOT = EXECUTION_ROOT / "runs" +AUTHORIZATION_ROOT = EXECUTION_ROOT / "authorizations" +FREEZE_PATH = EXECUTION_ROOT / "freeze.json" +SHAKEDOWN_PATH = EXECUTION_ROOT / "native-delivery-shakedown.json" +CLAUDE_WRAPPER_PATH = ( + EXECUTION_ROOT / "provider-wrappers" / "claude" / ".claude-plugin" / "plugin.json" +) +EXTERNAL_ROOT = Path( + os.environ.get("TRACEBOOK_V2_EXTERNAL_ROOT", "/private/tmp/tracebook-agent-qualification-v2") +) + +PROTOCOL_ID = cohort.PROTOCOL_ID +FREEZE_KIND = "agent-qualification-generalization-execution-freeze-v1" +SHAKEDOWN_KIND = "agent-qualification-generalization-native-skill-shakedown-v1" +VERDICT_KIND = "agent-qualification-generalization-pre-score-verdict-v1" +AUTHORIZATION_KIND = "agent-qualification-generalization-provider-authorization-v1" +SKILL_NAME = "tracebook-qualify-matching-engine" +SYNTHETIC_SKILL_NAME = "native-delivery-probe" +SYNTHETIC_MARKER = "NATIVE_SKILL_PROBE_V2_20260801_7F31C9" +DEFAULT_TIMEOUT_SECONDS = 120 * 60 +EXPECTED_MODELS = {"codex": "gpt-5.6-sol", "claude": "claude-opus-4-8"} +PROVIDERS = tuple(EXPECTED_MODELS) + +_PLUGIN_MANIFEST = { + "author": {"name": "Tracebook evaluator"}, + "description": "Isolated carrier for a preregistered agent evaluation treatment.", + "name": "agent-qualification-treatment", + "version": "2.0.0", +} +_SYNTHETIC_SKILL = ( + "---\n" + "name: native-delivery-probe\n" + "description: Synthetic probe used only to verify native skill delivery.\n" + "---\n\n" + "When explicitly asked to run this probe, reply with exactly " + SYNTHETIC_MARKER + ".\n" +).encode() + +_ORIGINAL_CODEX_SHELL_ENVIRONMENT = helpers._codex_shell_environment + + +def _json_bytes(value: Mapping[str, Any]) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _read_json(path: Path, description: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise ExecutionError(f"cannot read {description} from {path}: {exc}") from exc + if not isinstance(value, dict): + raise ExecutionError(f"{description} must contain one JSON object") + return value + + +def _write_new_json(path: Path, value: Mapping[str, Any], description: str) -> None: + if path.exists() or path.is_symlink(): + raise ExecutionError(f"refusing to overwrite {description} {path}") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(_json_bytes(value)) + + +def _file_record(path: Path) -> dict[str, Any]: + resolved = path.resolve(strict=True) + if not resolved.is_file(): + raise ExecutionError(f"required file is unavailable: {path}") + try: + label = resolved.relative_to(REPOSITORY_ROOT.resolve()).as_posix() + except ValueError: + label = str(resolved) + return { + "path": label, + "sha256": helpers._sha256_file(resolved), + "bytes": resolved.stat().st_size, + } + + +def _validate_file_record(record: Any, description: str) -> Path: + if not isinstance(record, dict): + raise ExecutionError(f"{description} binding is missing") + raw_path = record.get("path") + if not isinstance(raw_path, str): + raise ExecutionError(f"{description} path is missing") + candidate = Path(raw_path) + path = candidate if candidate.is_absolute() else REPOSITORY_ROOT / candidate + if _file_record(path) != record: + raise ExecutionError(f"{description} binding drifted") + return path.resolve() + + +def _provider_binary(provider: str) -> Path: + if provider not in PROVIDERS: + raise ExecutionError(f"unknown provider {provider!r}") + override = os.environ.get(f"TRACEBOOK_V2_{provider.upper()}_BINARY") + candidates = [ + Path(override) if override else None, + Path(shutil.which(provider) or "") if shutil.which(provider) else None, + Path.home() / ".local" / "bin" / provider, + Path("/Applications/ChatGPT.app/Contents/Resources/codex") if provider == "codex" else None, + ] + for candidate in candidates: + if candidate is not None and candidate.is_file(): + return candidate.resolve() + raise ExecutionError(f"{provider} CLI is unavailable") + + +def _clean_probe_environment() -> dict[str, str]: + secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + return { + name: value + for name, value in os.environ.items() + if name not in helpers.SECRET_ENVIRONMENT_NAMES + and not any(fragment in name.upper() for fragment in secret_fragments) + } + + +def _provider_binding(provider: str) -> dict[str, Any]: + binary = _provider_binary(provider) + version = helpers._binary_version(binary, _clean_probe_environment()) + binding: dict[str, Any] = { + "configured_path": str(binary), + "resolved_path": str(binary.resolve()), + "sha256": helpers._sha256_file(binary), + "bytes": binary.stat().st_size, + "cli_version": version, + "requested_model": EXPECTED_MODELS[provider], + } + if provider == "claude": + completed = subprocess.run( + (str(binary), "auth", "status"), + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=30, + ) + try: + auth = json.loads(completed.stdout) + except json.JSONDecodeError: + auth = {"raw_sha256": hashlib.sha256(completed.stdout.encode()).hexdigest()} + if isinstance(auth, dict): + # Do not persist account identifiers in the experiment archive. + binding["auth_route"] = { + key: auth.get(key) + for key in ("loggedIn", "authMethod", "apiProvider", "subscriptionType") + if key in auth + } + return binding + + +def _validated_provider_bindings() -> dict[str, dict[str, Any]]: + bindings = {provider: _provider_binding(provider) for provider in PROVIDERS} + route = bindings["claude"].get("auth_route") + if ( + not isinstance(route, dict) + or route.get("loggedIn") is not True + or route.get("apiProvider") != "firstParty" + ): + raise ExecutionError( + "Claude must use an authenticated first-party route before native-delivery " + "shakedown or execution freeze" + ) + return bindings + + +def _configure_delivery() -> None: + helpers.CODEX_BINARY = _provider_binary("codex") + helpers.CLAUDE_BINARY = _provider_binary("claude") + helpers.CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json" + delivery.DEFAULT_CLAUDE_WRAPPER_PATH = CLAUDE_WRAPPER_PATH + delivery.EXPECTED_MODELS = dict(EXPECTED_MODELS) + helpers._codex_shell_environment = _codex_shell_environment + + +def _toolchain_environment() -> dict[str, str]: + values: dict[str, str] = {} + path_prefixes: list[str] = [] + for name in ("TRACEBOOK_V2_MAVEN_HOME", "TRACEBOOK_V2_JAVA_HOME"): + raw = os.environ.get(name) + if not raw: + continue + root = Path(raw).resolve() + if not root.is_dir(): + raise ExecutionError(f"{name} is not an available directory: {root}") + path_prefixes.append(str(root / "bin")) + if name == "TRACEBOOK_V2_JAVA_HOME": + values["JAVA_HOME"] = str(root) + python_home = os.environ.get("TRACEBOOK_V2_PYTHON_HOME") + if python_home: + root = Path(python_home).resolve() + if not root.is_dir(): + raise ExecutionError(f"TRACEBOOK_V2_PYTHON_HOME is not an available directory: {root}") + path_prefixes.append(str(root / "bin")) + uv_binary = os.environ.get("TRACEBOOK_V2_UV_BINARY") + if uv_binary: + path = Path(uv_binary).resolve() + if not path.is_file(): + raise ExecutionError(f"TRACEBOOK_V2_UV_BINARY is not an available file: {path}") + path_prefixes.append(str(path.parent)) + cargo_bin = Path.home() / ".cargo" / "bin" + if cargo_bin.is_dir(): + path_prefixes.append(str(cargo_bin)) + values["PATH"] = ":".join( + [*path_prefixes, os.environ.get("PATH", "/usr/bin:/bin:/usr/sbin:/sbin")] + ) + return values + + +def _codex_shell_environment(scratch: Path) -> dict[str, str]: + values = _ORIGINAL_CODEX_SHELL_ENVIRONMENT(scratch) + values.update(_toolchain_environment()) + values.update(_fresh_environment(scratch)) + return values + + +def _claude_settings(workspace: Path, scratch: Path, plugin_root: Path) -> dict[str, Any]: + settings = delivery._claude_treatment_settings(workspace, scratch, plugin_root) + settings["env"].update(_toolchain_environment()) + settings["env"].update(_fresh_environment(scratch)) + return settings + + +def _fresh_environment(scratch: Path) -> dict[str, str]: + return { + "FRESH_M2": str(scratch / "m2-repository"), + "FRESH_CARGO_HOME": str(scratch / "cargo-home"), + "FRESH_TARGET": str(scratch / "cargo-target"), + } + + +def _write_wrapper() -> None: + if CLAUDE_WRAPPER_PATH.exists(): + if _read_json(CLAUDE_WRAPPER_PATH, "Claude plugin wrapper") != _PLUGIN_MANIFEST: + raise ExecutionError("Claude plugin wrapper drifted") + return + CLAUDE_WRAPPER_PATH.parent.mkdir(parents=True, exist_ok=True) + CLAUDE_WRAPPER_PATH.write_bytes(_json_bytes(_PLUGIN_MANIFEST)) + + +def _case_index(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + cases = manifest.get("cases") + if not isinstance(cases, list): + raise ExecutionError("validated cohort lost its cases") + return { + str(case["id"]): dict(case) + for case in cases + if isinstance(case, dict) and isinstance(case.get("id"), str) + } + + +def _plan_entry(plan: Mapping[str, Any], run_id: str) -> dict[str, Any]: + entries = plan.get("entries") + if not isinstance(entries, list): + raise ExecutionError("validated cohort lost its plan entries") + matches = [entry for entry in entries if isinstance(entry, dict) and entry.get("id") == run_id] + if len(matches) != 1: + raise ExecutionError(f"run {run_id!r} is not one frozen plan entry") + return dict(matches[0]) + + +def _load_primary() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + cohort.validate() + return ( + cohort._load_json(cohort.MANIFEST_PATH), + cohort._load_json(cohort.PLAN_PATH), + cohort._load_json(cohort.FREEZE_PATH), + ) + + +def _authorization_path(provider: str) -> Path: + if provider not in PROVIDERS: + raise ExecutionError(f"unknown provider {provider!r}") + return AUTHORIZATION_ROOT / f"{provider}.json" + + +def record_authorization(provider: str, statement_file: Path) -> dict[str, Any]: + """Record explicit consent already supplied by the user; never infer it.""" + + validate_freeze() + statement = statement_file.read_text().strip() + if not statement: + raise ExecutionError("authorization statement must not be empty") + payload = { + "kind": AUTHORIZATION_KIND, + "protocol": PROTOCOL_ID, + "provider": provider, + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + "user_statement": statement, + "statement_sha256": hashlib.sha256(statement.encode()).hexdigest(), + "recorded_at": helpers._utc_now(), + } + _write_new_json(_authorization_path(provider), payload, f"{provider} authorization") + return payload + + +def validate_authorization(provider: str) -> dict[str, Any]: + payload = _read_json(_authorization_path(provider), f"{provider} authorization") + expected = { + "kind": AUTHORIZATION_KIND, + "protocol": PROTOCOL_ID, + "provider": provider, + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + } + for field, value in expected.items(): + if payload.get(field) != value: + raise ExecutionError(f"{provider} authorization {field!r} is invalid") + statement = payload.get("user_statement") + if not isinstance(statement, str) or not statement.strip(): + raise ExecutionError(f"{provider} authorization statement is missing") + if payload.get("statement_sha256") != hashlib.sha256(statement.encode()).hexdigest(): + raise ExecutionError(f"{provider} authorization statement binding drifted") + if not isinstance(payload.get("recorded_at"), str): + raise ExecutionError(f"{provider} authorization timestamp is missing") + return payload + + +def _toolchain_versions() -> dict[str, Any]: + result: dict[str, Any] = {} + environment = dict(os.environ) + environment.update(_toolchain_environment()) + for name, command in { + "git": ("git", "--version"), + "python": (sys.executable, "--version"), + "java": ("java", "-version"), + "maven": ("mvn", "--version"), + "cargo": ("cargo", "--version"), + "rustc": ("rustc", "--version"), + }.items(): + binary = ( + shutil.which(command[0], path=environment["PATH"]) + if not Path(command[0]).is_absolute() + else command[0] + ) + if not binary: + result[name] = {"available": False} + continue + completed = subprocess.run( + (str(binary), *command[1:]), + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=30, + env=environment, + ) + result[name] = { + "available": completed.returncode == 0, + "path": str(Path(binary).resolve()), + "version": completed.stdout.strip().splitlines()[0] if completed.stdout.strip() else "", + } + return result + + +def _native_surface( + *, + provider: str, + condition: str, + workspace: Path, + run_root: Path, + scratch: Path, + environment: dict[str, str], + skill: bytes, + skill_name: str, +) -> tuple[list[str], dict[str, Any], Path | None]: + _configure_delivery() + return delivery._prepare_native_surface( + agent=provider, + condition=condition, + workspace=workspace, + external_run_root=run_root, + scratch=scratch, + environment=environment, + skill_content=skill, + skill_name=skill_name, + ) + + +def _claude_command(settings_path: Path, plugin_root: Path) -> list[str]: + command = delivery._claude_treatment_command(settings_path, plugin_root) + return command + + +def _run_delivery_probe(provider: str, condition: str, root: Path) -> dict[str, Any]: + workspace = root / "workspace" + scratch = root / "scratch" + workspace.mkdir(parents=True) + scratch.mkdir(parents=True) + (workspace / "README.md").write_text("Synthetic native skill delivery probe.\n") + environment = helpers._clean_agent_environment(scratch) + environment.update(_toolchain_environment()) + command, surface, plugin_root = _native_surface( + provider=provider, + condition=condition, + workspace=workspace, + run_root=root, + scratch=scratch, + environment=environment, + skill=_SYNTHETIC_SKILL, + skill_name=SYNTHETIC_SKILL_NAME, + ) + if provider == "claude": + if plugin_root is None: + raise ExecutionError("Claude probe plugin was not prepared") + settings_path = root / "claude-settings.json" + helpers._write_json(settings_path, _claude_settings(workspace, scratch, plugin_root)) + command = _claude_command(settings_path, plugin_root) + transcript = root / f"{provider}.jsonl" + stderr = root / f"{provider}.stderr" + prompt = ( + "Run the native-delivery-probe shakedown. If its native skill is available, " + "use it. If unavailable, reply exactly NO_NATIVE_DELIVERY. Do not search for " + "skill files or inspect any path not supplied by the native skill catalog." + ) + exit_code, timed_out, elapsed = helpers._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript, + stderr_path=stderr, + timeout_seconds=600, + ) + audit = delivery._audit_native_surface( + agent=provider, + condition=condition, + external_run_root=root, + scratch=scratch, + skill_content=_SYNTHETIC_SKILL, + skill_name=SYNTHETIC_SKILL_NAME, + ) + terminal = delivery._transcript_terminal_state(transcript, provider) + result = delivery._transcript_result_text(transcript, provider) + expected = SYNTHETIC_MARKER if condition == "skill" else "NO_NATIVE_DELIVERY" + errors = [] + if exit_code != 0 or timed_out or terminal != "completed": + errors.append("provider did not complete cleanly") + if audit.get("mutated") is not False: + errors.append("native skill surface mutated") + if result != expected: + errors.append(f"result {result!r} != {expected!r}") + return { + "valid": not errors, + "errors": errors, + "condition": condition, + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed, + "terminal_state": terminal, + "result": result, + "surface": audit, + "transcript": _file_record(transcript), + "stderr": _file_record(stderr), + } + + +def shakedown() -> dict[str, Any]: + cohort.validate() + _write_wrapper() + provider_bindings = _validated_provider_bindings() + if SHAKEDOWN_PATH.exists() or SHAKEDOWN_PATH.is_symlink(): + raise ExecutionError(f"refusing to overwrite shakedown {SHAKEDOWN_PATH}") + root = EXECUTION_ROOT / "shakedown-work" + if root.exists() or root.is_symlink(): + raise ExecutionError(f"refusing to reuse shakedown work root {root}") + providers: dict[str, Any] = {} + for provider in PROVIDERS: + conditions: dict[str, Any] = {} + for condition in ("docs", "skill"): + result = _run_delivery_probe(provider, condition, root / provider / condition) + if result["valid"] is not True: + raise ExecutionError( + f"{provider} {condition} native-delivery probe failed: " + + "; ".join(result["errors"]) + ) + conditions[condition] = result + providers[provider] = {"passed": True, "conditions": conditions} + verdict = { + "kind": SHAKEDOWN_KIND, + "protocol": PROTOCOL_ID, + "created_at": helpers._utc_now(), + "passed": True, + "runner": _file_record(RUNNER_PATH), + "synthetic_skill_sha256": hashlib.sha256(_SYNTHETIC_SKILL).hexdigest(), + "marker": SYNTHETIC_MARKER, + "provider_bindings": provider_bindings, + "providers": providers, + } + _write_new_json(SHAKEDOWN_PATH, verdict, "native-delivery shakedown") + return verdict + + +def _validate_shakedown() -> dict[str, Any]: + verdict = _read_json(SHAKEDOWN_PATH, "native-delivery shakedown") + if verdict.get("kind") != SHAKEDOWN_KIND or verdict.get("protocol") != PROTOCOL_ID: + raise ExecutionError("native-delivery shakedown identity changed") + if verdict.get("passed") is not True or verdict.get("runner") != _file_record(RUNNER_PATH): + raise ExecutionError("native-delivery shakedown binding changed") + if verdict.get("synthetic_skill_sha256") != hashlib.sha256(_SYNTHETIC_SKILL).hexdigest(): + raise ExecutionError("native-delivery synthetic skill changed") + if verdict.get("marker") != SYNTHETIC_MARKER: + raise ExecutionError("native-delivery marker changed") + if verdict.get("provider_bindings") != { + provider: _provider_binding(provider) for provider in PROVIDERS + }: + raise ExecutionError("native-delivery provider bindings changed") + providers = verdict.get("providers") + if not isinstance(providers, dict) or set(providers) != set(PROVIDERS): + raise ExecutionError("native-delivery provider coverage changed") + for provider in PROVIDERS: + conditions = ( + providers[provider].get("conditions") if isinstance(providers[provider], dict) else None + ) + if not isinstance(conditions, dict) or set(conditions) != {"docs", "skill"}: + raise ExecutionError(f"{provider} shakedown conditions changed") + if any( + result.get("valid") is not True + for result in conditions.values() + if isinstance(result, dict) + ): + raise ExecutionError(f"{provider} shakedown did not pass") + for result in conditions.values(): + if not isinstance(result, dict): + raise ExecutionError(f"{provider} shakedown result is invalid") + _validate_file_record(result.get("transcript"), f"{provider} transcript") + _validate_file_record(result.get("stderr"), f"{provider} stderr") + return verdict + + +def _freeze_payload(created_at: str) -> dict[str, Any]: + manifest, plan, primary_freeze = _load_primary() + del manifest, plan, primary_freeze + shakedown_verdict = _validate_shakedown() + inputs = { + "execution_addendum": _file_record(ADDENDUM_PATH), + "execution_runner": _file_record(RUNNER_PATH), + "execution_runner_tests": _file_record(TEST_PATH), + "helper_runner": _file_record(helpers.RUNNER_PATH), + "delivery_runner": _file_record(delivery.RUNNER_PATH), + "primary_protocol": _file_record(cohort.PROTOCOL_PATH), + "primary_prompt": _file_record(cohort.PROMPT_PATH), + "primary_runner": _file_record(cohort.RUNNER_PATH), + "primary_manifest": _file_record(cohort.MANIFEST_PATH), + "primary_plan": _file_record(cohort.PLAN_PATH), + "primary_freeze": _file_record(cohort.FREEZE_PATH), + "skill": _file_record(cohort.SKILL_PATH), + "claude_plugin_wrapper": _file_record(CLAUDE_WRAPPER_PATH), + "native_delivery_shakedown": _file_record(SHAKEDOWN_PATH), + } + return { + "kind": FREEZE_KIND, + "protocol": PROTOCOL_ID, + "created_at": created_at, + "host": { + "hostname": socket.gethostname(), + "platform": platform.platform(), + "machine": platform.machine(), + }, + "inputs": inputs, + "provider_bindings": shakedown_verdict["provider_bindings"], + "toolchains": _toolchain_versions(), + "run_count": 24, + "order": "exact seeded primary plan order", + "authorization": "provider-specific external records required after freeze", + "restart_policy": ( + "archive and retry only genuine technical interruptions without a completed " + "provider turn; never retry a timeout, refusal, or semantic outcome" + ), + } + + +def freeze_execution() -> dict[str, Any]: + if FREEZE_PATH.exists() or FREEZE_PATH.is_symlink(): + raise ExecutionError(f"refusing to overwrite execution freeze {FREEZE_PATH}") + payload = _freeze_payload(helpers._utc_now()) + _write_new_json(FREEZE_PATH, payload, "execution freeze") + return payload + + +def validate_freeze() -> dict[str, Any]: + cohort.validate() + payload = _read_json(FREEZE_PATH, "execution freeze") + if payload.get("kind") != FREEZE_KIND or payload.get("protocol") != PROTOCOL_ID: + raise ExecutionError("execution freeze identity changed") + created_at = payload.get("created_at") + if not isinstance(created_at, str) or not created_at: + raise ExecutionError("execution freeze timestamp is missing") + if payload != _freeze_payload(created_at): + raise ExecutionError("execution freeze content changed") + return payload + + +def _copy_fixture(case: Mapping[str, Any], scratch: Path) -> dict[str, Any]: + declaration = case.get("dependency_cache") + if not isinstance(declaration, dict): + raise ExecutionError("case dependency-cache declaration is missing") + source = (REPOSITORY_ROOT / str(declaration["source"])).resolve(strict=True) + target_name = "m2-repository" if case["id"] == "c4-compatible" else "cargo-home" + target = scratch / target_name + helpers._copy_frozen_tree(source, target, str(declaration["tree_sha256"])) + return { + "source_manifest_sha256": declaration["manifest_sha256"], + "initial_tree_sha256": declaration["tree_sha256"], + "target": str(target), + } + + +def _record_fixture_final(fixture: dict[str, Any]) -> None: + target = Path(str(fixture["target"])) + if not target.is_dir(): + fixture["final_state"] = "missing" + return + fixture["final_state"] = "present" + fixture["final_tree_sha256"] = helpers._snapshot_digest(target) + + +def _run_root(run_id: str) -> Path: + return RUNS_ROOT / run_id + + +def _external_run_root(run_id: str) -> Path: + return EXTERNAL_ROOT / run_id + + +def _technical_verdict( + run_root: Path, metadata: Mapping[str, Any], errors: list[str] +) -> dict[str, Any]: + verdict = { + "kind": VERDICT_KIND, + "protocol": PROTOCOL_ID, + "run_id": metadata.get("run_id"), + "valid": not errors, + "errors": errors, + "terminal_state": metadata.get("terminal_state"), + "timed_out": metadata.get("timed_out"), + "exit_code": metadata.get("exit_code"), + "metadata_sha256": helpers._sha256_file(run_root / "metadata.json"), + "transcript_sha256": metadata.get("transcript_sha256"), + "created_at": helpers._utc_now(), + } + _write_new_json(run_root / "pre-score-verdict.json", verdict, "pre-score verdict") + return verdict + + +def validate_run(run_id: str) -> dict[str, Any]: + freeze = validate_freeze() + manifest, plan, _ = _load_primary() + entry = _plan_entry(plan, run_id) + run_root = _run_root(run_id) + metadata = _read_json(run_root / "metadata.json", "run metadata") + verdict = _read_json(run_root / "pre-score-verdict.json", "pre-score verdict") + errors: list[str] = [] + expected = { + "protocol": PROTOCOL_ID, + "run_id": run_id, + "case_id": entry["case_id"], + "provider": entry["agent"], + "condition": entry["condition"], + "repetition": entry["repetition"], + "order": entry["order"], + "primary_manifest_sha256": helpers._sha256_file(cohort.MANIFEST_PATH), + "primary_plan_sha256": helpers._sha256_file(cohort.PLAN_PATH), + "primary_freeze_sha256": helpers._sha256_file(cohort.FREEZE_PATH), + "execution_freeze_sha256": helpers._sha256_file(FREEZE_PATH), + "execution_runner_sha256": helpers._sha256_file(RUNNER_PATH), + } + for field, value in expected.items(): + if metadata.get(field) != value: + errors.append(f"metadata {field} binding changed") + case = _case_index(manifest)[str(entry["case_id"])] + if metadata.get("snapshot_sha256") != case["snapshot_sha256"]: + errors.append("candidate snapshot binding changed") + if metadata.get("initial_workspace_sha256") != case["snapshot_sha256"]: + errors.append("initial workspace binding changed") + if metadata.get("frozen_skill_sha256") != helpers._sha256_file(cohort.SKILL_PATH): + errors.append("frozen skill metadata binding changed") + transcript = run_root / f"{entry['agent']}.jsonl" + stderr = run_root / f"{entry['agent']}.stderr" + prompt = run_root / "prompt.txt" + for artifact in (transcript, stderr, prompt): + if not artifact.is_file() or artifact.is_symlink(): + errors.append(f"missing required artifact {artifact.name}") + if transcript.is_file() and metadata.get("transcript_sha256") != helpers._sha256_file( + transcript + ): + errors.append("transcript hash changed") + if stderr.is_file() and metadata.get("stderr_sha256") != helpers._sha256_file(stderr): + errors.append("stderr hash changed") + rendered = cohort.render_prompt(case, str(entry["condition"])) + rendered_hash = hashlib.sha256(rendered.encode()).hexdigest() + if metadata.get("rendered_prompt_sha256") != rendered_hash: + errors.append("rendered prompt metadata binding changed") + if prompt.is_file() and helpers._sha256_file(prompt) != rendered_hash: + errors.append("rendered prompt artifact changed") + frozen_provider = freeze["provider_bindings"][entry["agent"]] + if metadata.get("provider_binding") != frozen_provider: + errors.append("provider binding changed") + if metadata.get("authorization_sha256") != helpers._sha256_file( + _authorization_path(str(entry["agent"])) + ): + errors.append("provider authorization binding changed") + if metadata.get("gold_manifest_sha256") != case["gold_sha256"]: + errors.append("gold manifest binding changed") + if metadata.get("model_identifier") != EXPECTED_MODELS[str(entry["agent"])]: + errors.append("reported model identifier changed") + if metadata.get("exit_code") != 0: + errors.append("provider subprocess exit code changed") + if metadata.get("terminal_state") != "completed" or metadata.get("timed_out") is not False: + errors.append("provider did not complete a valid technical turn") + surface = metadata.get("native_surface") + expected_skill = entry["condition"] == "skill" + if not isinstance(surface, dict) or surface.get("skill_installed") is not expected_skill: + errors.append("native skill surface condition changed") + else: + expected_skill_hash = helpers._sha256_file(cohort.SKILL_PATH) if expected_skill else None + if surface.get("semantic_file_count") != (1 if expected_skill else 0): + errors.append("native skill semantic file count changed") + if surface.get("skill_sha256") != expected_skill_hash: + errors.append("native skill hash changed") + if ( + not isinstance(surface.get("final_audit"), dict) + or surface["final_audit"].get("mutated") is not False + ): + errors.append("native skill surface audit failed") + for artifact_name in ( + "git-status.txt", + "workspace.patch", + "workspace-files.json", + "workspace-final", + ): + if not (run_root / artifact_name).exists(): + errors.append(f"missing required artifact {artifact_name}") + if verdict.get("kind") != VERDICT_KIND or verdict.get("run_id") != run_id: + errors.append("pre-score verdict identity changed") + if verdict.get("metadata_sha256") != helpers._sha256_file(run_root / "metadata.json"): + errors.append("pre-score verdict metadata binding changed") + if verdict.get("transcript_sha256") != metadata.get("transcript_sha256"): + errors.append("pre-score verdict transcript binding changed") + if verdict.get("valid") is not True or verdict.get("errors") != []: + errors.append("pre-score verdict is not valid") + if errors: + raise ExecutionError("; ".join(errors)) + return verdict + + +def _validate_prior_runs(plan: Mapping[str, Any], order: int) -> None: + entries = plan.get("entries") + if not isinstance(entries, list): + raise ExecutionError("validated cohort lost its plan entries") + for prior in entries[: order - 1]: + if not isinstance(prior, dict): + raise ExecutionError("plan entry shape changed") + validate_run(str(prior["id"])) + + +def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> dict[str, Any]: + if timeout_seconds != DEFAULT_TIMEOUT_SECONDS: + raise ExecutionError(f"official timeout must remain {DEFAULT_TIMEOUT_SECONDS} seconds") + freeze = validate_freeze() + manifest, plan, _ = _load_primary() + entry = _plan_entry(plan, run_id) + provider = str(entry["agent"]) + validate_authorization(provider) + _validate_prior_runs(plan, int(entry["order"])) + case = _case_index(manifest)[str(entry["case_id"])] + run_root = _run_root(run_id) + external_root = _external_run_root(run_id) + if run_root.exists() or run_root.is_symlink(): + raise ExecutionError(f"refusing to overwrite existing run {run_root}") + if external_root.exists() or external_root.is_symlink(): + raise ExecutionError(f"refusing to reuse external workspace {external_root}") + + source = (REPOSITORY_ROOT / str(case["source"])).resolve(strict=True) + workspace = external_root / "workspace" + scratch = external_root / "scratch" + run_root.mkdir(parents=True) + scratch.mkdir(parents=True) + helpers._initialize_snapshot(source, workspace) + initial_workspace_hash = helpers._snapshot_digest(workspace) + if initial_workspace_hash != case["snapshot_sha256"]: + raise ExecutionError("fresh workspace changed the frozen candidate snapshot") + for relative in (".tracebook", ".eval-cache", "target"): + if (workspace / relative).exists() or (workspace / relative).is_symlink(): + raise ExecutionError(f"fresh workspace unexpectedly contains {relative}") + prompt = cohort.render_prompt(case, str(entry["condition"])) + (run_root / "prompt.txt").write_text(prompt) + environment = helpers._clean_agent_environment(scratch) + environment.update(_toolchain_environment()) + fixture = _copy_fixture(case, scratch) + environment.update(_fresh_environment(scratch)) + Path(environment["FRESH_TARGET"]).mkdir(exist_ok=True) + skill = cohort.SKILL_PATH.read_bytes() + command, surface, plugin_root = _native_surface( + provider=provider, + condition=str(entry["condition"]), + workspace=workspace, + run_root=external_root, + scratch=scratch, + environment=environment, + skill=skill, + skill_name=SKILL_NAME, + ) + if provider == "claude": + if plugin_root is None: + raise ExecutionError("Claude plugin was not prepared") + settings_path = run_root / "claude-settings.json" + helpers._write_json(settings_path, _claude_settings(workspace, scratch, plugin_root)) + command = _claude_command(settings_path, plugin_root) + + transcript = run_root / f"{provider}.jsonl" + stderr = run_root / f"{provider}.stderr" + binary = _provider_binary(provider) + metadata: dict[str, Any] = { + "protocol": PROTOCOL_ID, + "run_id": run_id, + "case_id": entry["case_id"], + "provider": provider, + "condition": entry["condition"], + "repetition": entry["repetition"], + "order": entry["order"], + "started_at": helpers._utc_now(), + "timeout_seconds": timeout_seconds, + "primary_manifest_sha256": helpers._sha256_file(cohort.MANIFEST_PATH), + "primary_plan_sha256": helpers._sha256_file(cohort.PLAN_PATH), + "primary_freeze_sha256": helpers._sha256_file(cohort.FREEZE_PATH), + "execution_freeze_sha256": helpers._sha256_file(FREEZE_PATH), + "execution_runner_sha256": helpers._sha256_file(RUNNER_PATH), + "snapshot_sha256": case["snapshot_sha256"], + "initial_workspace_sha256": initial_workspace_hash, + "gold_manifest_sha256": case["gold_sha256"], + "rendered_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "authorization_sha256": helpers._sha256_file(_authorization_path(provider)), + "provider_binding": freeze["provider_bindings"][provider], + "command": command, + "requested_model": EXPECTED_MODELS[provider], + "cli_version": helpers._binary_version(binary, environment), + "binary_sha256": helpers._sha256_file(binary), + "prompt_transport": "stdin", + "fixture": fixture, + "native_surface": surface, + "skill_injected": entry["condition"] == "skill", + "frozen_skill_sha256": helpers._sha256_file(cohort.SKILL_PATH), + } + helpers._write_json(run_root / "metadata.json", metadata) + exit_code, timed_out, elapsed = helpers._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript, + stderr_path=stderr, + timeout_seconds=timeout_seconds, + ) + surface["final_audit"] = delivery._audit_native_surface( + agent=provider, + condition=str(entry["condition"]), + external_run_root=external_root, + scratch=scratch, + skill_content=skill, + skill_name=SKILL_NAME, + ) + helpers._write_workspace_evidence(workspace, run_root) + _record_fixture_final(fixture) + shutil.copytree( + workspace, + run_root / "workspace-final", + symlinks=True, + ignore=shutil.ignore_patterns(".git", "target", "node_modules", ".eval-cache"), + ) + terminal = delivery._transcript_terminal_state(transcript, provider) + model, model_source = helpers._reported_model(transcript, provider) + metadata.update( + { + "completed_at": helpers._utc_now(), + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed, + "terminal_state": terminal, + "model_identifier": model, + "model_identifier_source": model_source, + "transcript_sha256": helpers._sha256_file(transcript), + "stderr_sha256": helpers._sha256_file(stderr), + "fixture": fixture, + "native_surface": surface, + } + ) + helpers._write_json(run_root / "metadata.json", metadata) + errors: list[str] = [] + if timed_out: + errors.append("provider subprocess timed out") + if terminal != "completed": + errors.append(f"provider terminal state was {terminal!r}") + if exit_code != 0: + errors.append(f"provider exit code was {exit_code}") + if model != EXPECTED_MODELS[provider]: + errors.append(f"reported model {model!r} != {EXPECTED_MODELS[provider]!r}") + if surface["final_audit"].get("mutated") is not False: + errors.append("native skill surface mutated") + verdict = _technical_verdict(run_root, metadata, errors) + if errors: + raise ExecutionError(f"run {run_id!r} is technically invalid: " + "; ".join(errors)) + validate_run(run_id) + shutil.rmtree(external_root) + return {"metadata": metadata, "verdict": verdict} + + +def status() -> dict[str, Any]: + cohort_summary = cohort.validate() + _, plan, _ = _load_primary() + completed = [] + for entry in plan["entries"]: + run_id = str(entry["id"]) + if (_run_root(run_id) / "pre-score-verdict.json").is_file(): + try: + validate_run(run_id) + except ExecutionError: + continue + completed.append(run_id) + authorizations = {provider: _authorization_path(provider).is_file() for provider in PROVIDERS} + next_run = next( + (entry["id"] for entry in plan["entries"] if entry["id"] not in completed), None + ) + return { + **cohort_summary, + "execution_frozen": FREEZE_PATH.is_file(), + "authorizations_recorded": authorizations, + "completed_valid_runs": len(completed), + "next_run": next_run, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("shakedown") + commands.add_parser("freeze") + commands.add_parser("validate") + commands.add_parser("status") + authorize = commands.add_parser("authorize") + authorize.add_argument("--provider", required=True, choices=PROVIDERS) + authorize.add_argument("--statement-file", required=True, type=Path) + run = commands.add_parser("run") + run.add_argument("--run-id", required=True) + run.add_argument("--timeout-seconds", type=int, default=DEFAULT_TIMEOUT_SECONDS) + validate_run_parser = commands.add_parser("validate-run") + validate_run_parser.add_argument("--run-id", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "shakedown": + payload = shakedown() + elif args.command == "freeze": + payload = freeze_execution() + elif args.command == "validate": + payload = validate_freeze() + elif args.command == "authorize": + payload = record_authorization(args.provider, args.statement_file) + elif args.command == "run": + payload = execute_run(args.run_id, args.timeout_seconds) + elif args.command == "validate-run": + payload = validate_run(args.run_id) + else: + payload = status() + except (ExecutionError, OSError, ValueError, subprocess.SubprocessError) as exc: + print(f"Execution experiment error: {exc}", file=sys.stderr) + return 2 + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/agent_qualification_prompt.txt b/experiments/agent_qualification_prompt.txt new file mode 100644 index 0000000..419565d --- /dev/null +++ b/experiments/agent_qualification_prompt.txt @@ -0,0 +1,51 @@ +You are the release-validation engineer for a small venue-infrastructure team +that owns the matching logic in this repository. + +CASE: {{case_id}} +REPOSITORY SNAPSHOT: {{snapshot_id}} +PINNED REVISION: {{revision}} +SUPPORTED BUILD/TEST COMMANDS: {{commands}} +OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}} +EXCLUDED SCOPE: {{excluded_scope}} + +Determine whether this pinned engine preserves the declared matching semantics. +Do not optimize for finding a bug. A correct result may be: + +- conformant behavior; +- a candidate-engine defect; +- an adapter or observation defect; +- a profile/contract mismatch; +- insufficient observability; or +- an ambiguity requiring maintainer adjudication. + +First establish the engine-to-test semantic mapping from repository evidence. +Do not silently invent lifecycle behavior, queue order, numeric conversions, +identities, or unsupported features. Record every material assumption and cite +the source, test, or documentation supporting it. + +Use any public development tool you can discover if it is appropriate. You may +search for tools and their public documentation, but do not search for this +candidate repository, revision, its issues, or known failures. Do not change +production matching behavior. Evaluation-only adapters, artifacts, and native +tests are allowed. + +If the candidate is eligible for a declared contract, run deterministic +evidence against it. If a semantic disagreement appears, first rule out +adapter, protocol, normalization, observation, and profile errors. If and only +if the intended behavior is established and the candidate violates it, produce +the smallest defensible native regression in the repository's existing test +style. + +Deliver: + +1. CLASSIFICATION.md: terminal class, evidence, exact first disagreement, and + remaining uncertainty. +2. MAPPING.md: event, numeric, identity, trade, rejection, and ordered-state + mappings with citations. +3. COMMANDS.log: every material command and exit code. +4. Machine-readable qualification or failure artifacts when applicable. +5. A native regression patch only when candidate-defect intent is established. +6. A concise maintainer handoff that makes no claim beyond the evidence. + +Do not commit, push, open an issue, or contact a maintainer. Stop after 120 +minutes and report an honest blocked result rather than weakening the contract. diff --git a/experiments/agent_qualification_treatments.py b/experiments/agent_qualification_treatments.py new file mode 100644 index 0000000..b023805 --- /dev/null +++ b/experiments/agent_qualification_treatments.py @@ -0,0 +1,1692 @@ +"""Frozen docs/skill treatment runner for the agent qualification evaluation. + +The baseline runner is a scored, hash-pinned input and must remain unchanged. +This companion imports its isolation and evidence helpers while adding only the +pre-registered docs and native-skill treatment surfaces. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import shlex +import shutil +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence +from urllib.parse import urlparse + +if __package__: + from . import agent_qualification as baseline +else: # Support ``python experiments/agent_qualification_treatments.py``. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import agent_qualification as baseline # type: ignore[no-redef] + +EvaluationError = baseline.EvaluationError + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +PRIVATE_ROOT = baseline.PRIVATE_ROOT +TREATMENT_ROOT = PRIVATE_ROOT / "treatments" +DEFAULT_SKILL_PATH = TREATMENT_ROOT / "tracebook-qualify-matching-engine" / "SKILL.md" +DEFAULT_EXTERNAL_PINS_PATH = TREATMENT_ROOT / "EXTERNAL_PINS.json" +DEFAULT_PROTOCOL_AMENDMENT_PATH = TREATMENT_ROOT / "TREATMENT_PROTOCOL.md" +DEFAULT_AUTHORIZATION_PATH = TREATMENT_ROOT / "AUTHORIZATION.json" +DEFAULT_CLAUDE_WRAPPER_PATH = ( + TREATMENT_ROOT / "provider-wrappers" / "claude" / ".claude-plugin" / "plugin.json" +) +DEFAULT_PLAN_PATH = TREATMENT_ROOT / "master-plan.json" +DEFAULT_FREEZE_PATH = TREATMENT_ROOT / "freeze.json" +DEFAULT_SHAKEDOWN_VERDICT_PATH = TREATMENT_ROOT / "native-delivery-shakedown.json" +DEFAULT_BASELINE_PLAN_PATH = baseline.DEFAULT_PLAN_PATH +EVALUATION_ROOT = PRIVATE_ROOT / "evaluation" +DEFAULT_RUBRIC_PATH = EVALUATION_ROOT / "rubric.json" +DEFAULT_VALIDATOR_PATH = EVALUATION_ROOT / "scorecard.py" +DEFAULT_BASELINE_SCORES_PATH = EVALUATION_ROOT / "scores" +DEFAULT_RUNS_PATH = baseline.DEFAULT_RUNS_PATH +DEFAULT_TEST_PATH = REPOSITORY_ROOT / "tests" / "test_agent_qualification_treatments.py" + +PROTOCOL_ID = baseline.PROTOCOL_ID +PLAN_KIND = "agent-qualification-treatment-master-plan-v1" +FREEZE_KIND = "agent-qualification-treatment-freeze-v1" +VERDICT_KIND = "agent-qualification-pre-score-run-verdict-v1" +SHAKEDOWN_KIND = "agent-qualification-native-skill-shakedown-v1" +TREATMENT_CONDITIONS = ("docs", "skill") +REPETITIONS = 3 +RUNS_PER_CONDITION = 12 +TOTAL_RUNS = 24 +RANDOMIZATION_SEED = baseline.RANDOMIZATION_SEED +SKILL_NAME = "tracebook-qualify-matching-engine" +EXPECTED_MODELS = { + "codex": "gpt-5.6-sol", + "claude": "claude-opus-4-8", +} + +MEASUREMENT_POLICY: dict[str, Any] = { + "assigned_payload_is_intervention": False, + "post_launch_evaluator_intervention": "I0", + "elapsed_endpoint": { + "name": "subprocess completion elapsed", + "definition": ( + "monotonic wall-clock seconds from provider subprocess spawn " "until termination" + ), + "not_measured": "time-to-first-valid-evidence", + }, + "timing_gate": { + "minimum_matched_autonomous_safe_pass_pairs": 3, + "both_cases_required": True, + "skill_safe_pass_rate_must_not_be_lower": True, + "paired_statistic": "median(skill_elapsed/docs_elapsed)", + "pass_threshold": 0.70, + "zero_c3_safety_violations_required": True, + "otherwise": ( + "timing is exploratory/inestimable; skill value may pass only " + "through at least +20 percentage points safe-pass rate" + ), + }, +} + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _json_bytes(value: Mapping[str, Any]) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _read_json_once(path: Path, description: str) -> tuple[dict[str, Any], bytes]: + try: + raw = path.read_bytes() + value = json.loads(raw) + except (OSError, json.JSONDecodeError) as exc: + raise EvaluationError(f"cannot read {description} from {path}: {exc}") from exc + if not isinstance(value, dict): + raise EvaluationError(f"{description} must contain one JSON object") + return value, raw + + +def _path_label(path: Path) -> str: + resolved = path.resolve() + try: + return resolved.relative_to(REPOSITORY_ROOT.resolve()).as_posix() + except ValueError: + return str(resolved) + + +def _reject_symlink_chain(path: Path, root: Path, description: str) -> None: + lexical_path = Path(os.path.abspath(path)) + lexical_root = Path(os.path.abspath(root)) + try: + relative = lexical_path.relative_to(lexical_root) + except ValueError as exc: + raise EvaluationError(f"{description} must stay inside {lexical_root}") from exc + cursor = lexical_root + if cursor.is_symlink(): + raise EvaluationError(f"{description} root must not be a symlink: {cursor}") + for part in relative.parts: + cursor /= part + if cursor.is_symlink(): + raise EvaluationError(f"{description} must not use symlinks: {cursor}") + + +def _private_file(path: Path, description: str) -> Path: + candidate = path if path.is_absolute() else REPOSITORY_ROOT / path + _reject_symlink_chain(candidate, PRIVATE_ROOT, description) + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(PRIVATE_ROOT.resolve(strict=True)) + except (OSError, ValueError) as exc: + raise EvaluationError(f"{description} must be an existing private file") from exc + mode = resolved.stat().st_mode + if not stat.S_ISREG(mode): + raise EvaluationError(f"{description} must be a regular file: {resolved}") + return resolved + + +def _private_output(path: Path, description: str) -> Path: + candidate = path if path.is_absolute() else REPOSITORY_ROOT / path + parent = candidate.parent + _reject_symlink_chain(parent, PRIVATE_ROOT, description) + parent.mkdir(parents=True, exist_ok=True) + try: + parent.resolve(strict=True).relative_to(PRIVATE_ROOT.resolve(strict=True)) + except (OSError, ValueError) as exc: + raise EvaluationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc + if candidate.exists() or candidate.is_symlink(): + raise EvaluationError(f"refusing to overwrite {description}: {candidate}") + return candidate + + +def _file_record(path: Path) -> dict[str, Any]: + return { + "path": _path_label(path), + "sha256": baseline._sha256_file(path), + "bytes": path.stat().st_size, + } + + +def _tree_record(root: Path) -> dict[str, Any]: + resolved = root.resolve(strict=True) + inventory = baseline._tree_inventory(resolved) + return { + "path": _path_label(resolved), + "sha256": baseline._snapshot_digest(resolved), + "file_count": len(inventory), + "files": inventory, + } + + +def _resolve_record( + record: Any, + description: str, + *, + expected_path: Path | None = None, + private: bool, +) -> tuple[Path, bytes]: + if not isinstance(record, dict): + raise EvaluationError(f"{description} record must be an object") + raw_path = record.get("path") + expected_hash = record.get("sha256") + expected_bytes = record.get("bytes") + if not isinstance(raw_path, str) or not isinstance(expected_hash, str): + raise EvaluationError(f"{description} record must pin path and sha256") + candidate = Path(raw_path) + if private: + path = _private_file(candidate, description) + else: + lexical_path = candidate if candidate.is_absolute() else REPOSITORY_ROOT / candidate + _reject_symlink_chain(lexical_path, REPOSITORY_ROOT, description) + path = lexical_path.resolve(strict=True) + if not path.is_file(): + raise EvaluationError(f"{description} must be a regular non-symlink file") + if expected_path is not None and path != expected_path.resolve(strict=True): + raise EvaluationError(f"{description} path changed: expected {expected_path}, got {path}") + raw = path.read_bytes() + actual_hash = _sha256_bytes(raw) + if actual_hash != expected_hash: + raise EvaluationError(f"{description} changed: expected {expected_hash}, got {actual_hash}") + if expected_bytes != len(raw): + raise EvaluationError(f"{description} byte length changed") + return path, raw + + +def _write_new_json(path: Path, payload: Mapping[str, Any], description: str) -> None: + target = _private_output(path, description) + target.write_bytes(_json_bytes(payload)) + + +def _collect_strings(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, dict): + result: list[str] = [] + for nested in value.values(): + result.extend(_collect_strings(nested)) + return result + if isinstance(value, list): + result = [] + for nested in value: + result.extend(_collect_strings(nested)) + return result + return [] + + +def _leakage_tokens(manifest: Mapping[str, Any]) -> set[str]: + tokens: set[str] = set() + cases = manifest.get("cases") + if not isinstance(cases, list): + return tokens + for case in cases: + if not isinstance(case, dict): + continue + selected: list[Any] = [ + case.get("id"), + case.get("snapshot_id"), + case.get("snapshot_sha256"), + case.get("revision"), + case.get("source"), + case.get("gold_manifest"), + case.get("gold_sha256"), + case.get("origin"), + ] + for raw in _collect_strings(selected): + normalized = raw.strip().casefold() + if len(normalized) >= 8: + tokens.add(normalized) + if "://" in raw: + parsed = urlparse(raw) + for part in parsed.path.split("/"): + part = part.strip().casefold() + if len(part) >= 8: + tokens.add(part) + return tokens + + +def _load_skill(skill_path: Path, manifest: Mapping[str, Any]) -> dict[str, Any]: + path = _private_file(skill_path, "skill") + raw = path.read_bytes() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise EvaluationError("skill must be UTF-8") from exc + if text.encode("utf-8") != raw: + raise EvaluationError("skill bytes must round-trip through UTF-8 exactly") + if not text.startswith("---\n") or f"name: {SKILL_NAME}\n" not in text: + raise EvaluationError(f"skill frontmatter must name {SKILL_NAME!r}") + lowered = text.casefold() + leaked = sorted(token for token in _leakage_tokens(manifest) if token in lowered) + if leaked: + raise EvaluationError( + "skill leakage audit found held-out identifier(s): " + ", ".join(leaked) + ) + return { + **_file_record(path), + "name": SKILL_NAME, + "content": raw, + } + + +def _ready_case_ids(manifest: Mapping[str, Any]) -> list[str]: + return sorted( + case_id + for case_id, case in baseline._case_index(manifest).items() + if case.get("status") == "ready" + ) + + +def _structural_cells(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: + cells = [ + { + "case_id": case_id, + "agent": agent, + "repetition": repetition, + } + for case_id in _ready_case_ids(manifest) + for agent in ("codex", "claude") + for repetition in range(1, REPETITIONS + 1) + ] + random.Random(RANDOMIZATION_SEED).shuffle(cells) + if len(cells) != RUNS_PER_CONDITION: + raise EvaluationError( + f"treatment protocol requires {RUNS_PER_CONDITION} cells, got {len(cells)}" + ) + return cells + + +def _evidence_id(ordinal: int) -> str: + material = f"{PROTOCOL_ID}|treatment|{RANDOMIZATION_SEED}|{ordinal}".encode() + return "ev_" + _sha256_bytes(material)[:20] + + +def _master_entries(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: + cells = _structural_cells(manifest) + entries: list[dict[str, Any]] = [] + ordinal = 0 + for condition in TREATMENT_CONDITIONS: + for cell in cells: + ordinal += 1 + run_id = f"{condition}__{cell['case_id']}__{cell['agent']}__r{cell['repetition']}" + entries.append( + { + **cell, + "condition": condition, + "run_id": run_id, + "evidence_id": _evidence_id(ordinal), + "ordinal": ordinal, + } + ) + return entries + + +def render_prompt(case: Mapping[str, Any]) -> str: + """Render the byte-identical user prompt used by both treatment arms.""" + + return baseline.render_prompt(case, "docs") + + +def _binary_binding(agent: str) -> dict[str, Any]: + binary = baseline.CODEX_BINARY if agent == "codex" else baseline.CLAUDE_BINARY + if not binary.is_file(): + raise EvaluationError(f"{agent} binary is unavailable at {binary}") + secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + clean_environment = { + name: value + for name, value in os.environ.items() + if name not in baseline.SECRET_ENVIRONMENT_NAMES + and not any(fragment in name.upper() for fragment in secret_fragments) + } + version = baseline._binary_version(binary, clean_environment) + return { + "configured_path": str(binary), + "resolved_path": str(binary.resolve()), + "sha256": baseline._sha256_file(binary), + "bytes": binary.stat().st_size, + "cli_version": version, + "requested_model": EXPECTED_MODELS[agent], + "required_reported_model": EXPECTED_MODELS[agent], + } + + +def _load_external_pins(path: Path) -> tuple[dict[str, Any], dict[str, Any]]: + resolved = _private_file(path, "external pins") + payload, _ = _read_json_once(resolved, "external pins") + if payload.get("protocol") != PROTOCOL_ID: + raise EvaluationError("external pins protocol changed") + models = payload.get("models") + release = payload.get("public_release") + if not isinstance(models, dict) or not isinstance(release, dict): + raise EvaluationError("external pins must contain models and public_release") + for agent in ("codex", "claude"): + expected = _binary_binding(agent) + pinned = models.get(agent) + if not isinstance(pinned, dict): + raise EvaluationError(f"external pins are missing {agent}") + checks = { + "binary_path": expected["configured_path"], + "binary_sha256": expected["sha256"], + "cli_version": expected["cli_version"], + "requested_model": expected["requested_model"], + "required_reported_model": expected["required_reported_model"], + } + for field, expected_value in checks.items(): + if pinned.get(field) != expected_value: + raise EvaluationError( + f"external {agent} pin {field!r} changed: " + f"expected {expected_value!r}, got {pinned.get(field)!r}" + ) + git = release.get("git") + pypi = release.get("pypi") + if not isinstance(git, dict) or not isinstance(pypi, dict): + raise EvaluationError("external release pins must contain git and pypi") + required_release_values = { + ("git", "tag"): "v0.6.0", + ("git", "tag_commit"): "d7e20a345974f8fb57537512962e283f14ae51e3", + ("pypi", "project"): "tracebook-conformance", + ("pypi", "version"): "0.6.0", + } + sections = {"git": git, "pypi": pypi} + for (section, field), expected_value in required_release_values.items(): + if sections[section].get(field) != expected_value: + raise EvaluationError( + f"public release pin {section}.{field} must be {expected_value!r}" + ) + for section, fields in { + "git": ("tagged_readme_sha256", "tagged_conformance_docs_sha256"), + "pypi": ( + "wheel_filename", + "wheel_sha256", + "sdist_filename", + "sdist_sha256", + ), + }.items(): + for field in fields: + value = sections[section].get(field) + if not isinstance(value, str) or not value: + raise EvaluationError(f"public release pin {section}.{field} is missing") + return payload, _file_record(resolved) + + +def _prompt_hashes( + manifest: Mapping[str, Any], +) -> dict[str, dict[str, str]]: + result: dict[str, dict[str, str]] = {} + for case_id in _ready_case_ids(manifest): + prompt = render_prompt(baseline._case_index(manifest)[case_id]) + prompt_hash = _sha256_bytes(prompt.encode()) + result[case_id] = {"docs": prompt_hash, "skill": prompt_hash} + return result + + +def _plan_payload( + manifest_path: Path, + manifest: Mapping[str, Any], + skill: Mapping[str, Any], + external_pins: Mapping[str, Any], + external_pins_record: Mapping[str, Any], + protocol_amendment_path: Path, + authorization_path: Path, + claude_wrapper_path: Path, +) -> dict[str, Any]: + entries = _master_entries(manifest) + prompt_hashes = _prompt_hashes(manifest) + for entry in entries: + entry["rendered_prompt_sha256"] = prompt_hashes[entry["case_id"]][entry["condition"]] + return { + "kind": PLAN_KIND, + "protocol": PROTOCOL_ID, + "created_at": baseline._utc_now(), + "randomization_seed": RANDOMIZATION_SEED, + "design": "docs-block-then-matched-skill-block", + "repetitions": REPETITIONS, + "runs_per_condition": RUNS_PER_CONDITION, + "timeout_seconds": baseline.DEFAULT_TIMEOUT_SECONDS, + "conditions": list(TREATMENT_CONDITIONS), + "manifest": _file_record(manifest_path), + "baseline_runner": _file_record(baseline.RUNNER_PATH), + "treatment_runner": _file_record(RUNNER_PATH), + "protocol_document": _file_record(baseline.PROTOCOL_PATH), + "prompt_template": _file_record(baseline.PROMPT_PATH), + "treatment_protocol": _file_record(protocol_amendment_path), + "authorization": _file_record(authorization_path), + "docs_treatment_sha256": _sha256_bytes(baseline.DOCS_TREATMENT.encode()), + "skill": {key: value for key, value in skill.items() if key != "content"}, + "native_wrappers": { + "codex": { + "kind": "isolated-codex-home-skill-v1", + "semantic_files": ["SKILL.md"], + }, + "claude": { + "kind": "isolated-claude-plugin-skill-v1", + "manifest": _file_record(claude_wrapper_path), + "source_tree": _tree_record(claude_wrapper_path.parents[1]), + "semantic_files": ["SKILL.md"], + }, + }, + "external_pins": dict(external_pins_record), + "pinned_models": external_pins["models"], + "public_release": external_pins["public_release"], + "rendered_prompt_sha256": prompt_hashes, + "measurement_policy": MEASUREMENT_POLICY, + "entries": entries, + } + + +def create_plan( + manifest_path: Path, + *, + skill_path: Path, + external_pins_path: Path, + protocol_amendment_path: Path, + authorization_path: Path, + claude_wrapper_path: Path, + output_path: Path, +) -> dict[str, Any]: + manifest_path = manifest_path.resolve(strict=True) + manifest = baseline.validate_manifest(manifest_path) + skill = _load_skill(skill_path, manifest) + external_pins, external_pins_record = _load_external_pins(external_pins_path) + amendment = _private_file(protocol_amendment_path, "treatment protocol") + authorization = _private_file(authorization_path, "authorization ledger") + wrapper = _private_file(claude_wrapper_path, "Claude plugin wrapper") + payload = _plan_payload( + manifest_path, + manifest, + skill, + external_pins, + external_pins_record, + amendment, + authorization, + wrapper, + ) + _write_new_json(output_path, payload, "master plan") + return payload + + +def _validate_plan_entry_shape(entries: Any) -> list[dict[str, Any]]: + if not isinstance(entries, list): + raise EvaluationError("master plan entries must be a list") + if len(entries) != TOTAL_RUNS: + raise EvaluationError(f"master plan must contain {TOTAL_RUNS} entries") + result: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + seen_evidence: set[str] = set() + for index, entry in enumerate(entries, start=1): + if not isinstance(entry, dict): + raise EvaluationError("every master plan entry must be an object") + if entry.get("ordinal") != index: + raise EvaluationError("master plan ordinals must be contiguous") + run_id = entry.get("run_id") + evidence_id = entry.get("evidence_id") + if not isinstance(run_id, str) or run_id in seen_ids: + raise EvaluationError("master plan run IDs must be unique strings") + if not isinstance(evidence_id, str) or evidence_id in seen_evidence: + raise EvaluationError("master plan evidence IDs must be unique opaque strings") + if entry.get("condition") not in TREATMENT_CONDITIONS: + raise EvaluationError("master plan contains an invalid condition") + seen_ids.add(run_id) + seen_evidence.add(evidence_id) + result.append(entry) + return result + + +def validate_plan( + plan_path: Path, + *, + manifest_path: Path, + skill_path: Path, + external_pins_path: Path, + protocol_amendment_path: Path, + authorization_path: Path, + claude_wrapper_path: Path, +) -> dict[str, Any]: + path = _private_file(plan_path, "master plan") + payload, raw = _read_json_once(path, "master plan") + manifest_path = manifest_path.resolve(strict=True) + manifest = baseline.validate_manifest(manifest_path) + skill = _load_skill(skill_path, manifest) + external_pins, external_record = _load_external_pins(external_pins_path) + amendment = _private_file(protocol_amendment_path, "treatment protocol") + authorization = _private_file(authorization_path, "authorization ledger") + wrapper = _private_file(claude_wrapper_path, "Claude plugin wrapper") + + if payload.get("kind") != PLAN_KIND or payload.get("protocol") != PROTOCOL_ID: + raise EvaluationError("master plan identity changed") + if payload.get("randomization_seed") != RANDOMIZATION_SEED: + raise EvaluationError("master plan randomization seed changed") + if payload.get("conditions") != list(TREATMENT_CONDITIONS): + raise EvaluationError("master plan condition order changed") + if payload.get("repetitions") != REPETITIONS: + raise EvaluationError("master plan repetition count changed") + if payload.get("timeout_seconds") != baseline.DEFAULT_TIMEOUT_SECONDS: + raise EvaluationError("master plan timeout changed") + expected_records = { + "manifest": _file_record(manifest_path), + "baseline_runner": _file_record(baseline.RUNNER_PATH), + "treatment_runner": _file_record(RUNNER_PATH), + "protocol_document": _file_record(baseline.PROTOCOL_PATH), + "prompt_template": _file_record(baseline.PROMPT_PATH), + "treatment_protocol": _file_record(amendment), + "authorization": _file_record(authorization), + "external_pins": external_record, + } + for field, expected in expected_records.items(): + if payload.get(field) != expected: + raise EvaluationError(f"master plan {field} binding changed") + expected_skill = {key: value for key, value in skill.items() if key != "content"} + if payload.get("skill") != expected_skill: + raise EvaluationError("master plan skill binding changed") + if payload.get("docs_treatment_sha256") != _sha256_bytes(baseline.DOCS_TREATMENT.encode()): + raise EvaluationError("master plan docs treatment binding changed") + if payload.get("pinned_models") != external_pins["models"]: + raise EvaluationError("master plan model pins changed") + if payload.get("public_release") != external_pins["public_release"]: + raise EvaluationError("master plan public release pins changed") + if payload.get("measurement_policy") != MEASUREMENT_POLICY: + raise EvaluationError("master plan measurement policy changed") + native_wrappers = payload.get("native_wrappers") + if not isinstance(native_wrappers, dict): + raise EvaluationError("master plan native wrapper bindings are missing") + claude_wrapper = native_wrappers.get("claude") + if not isinstance(claude_wrapper, dict) or claude_wrapper.get("manifest") != _file_record( + wrapper + ): + raise EvaluationError("master plan Claude wrapper binding changed") + if claude_wrapper.get("source_tree") != _tree_record(wrapper.parents[1]): + raise EvaluationError("master plan Claude wrapper tree changed") + if payload.get("rendered_prompt_sha256") != _prompt_hashes(manifest): + raise EvaluationError("master plan rendered prompt hashes changed") + + entries = _validate_plan_entry_shape(payload.get("entries")) + expected_entries = _master_entries(manifest) + prompt_hashes = _prompt_hashes(manifest) + for entry in expected_entries: + entry["rendered_prompt_sha256"] = prompt_hashes[entry["case_id"]][entry["condition"]] + if entries != expected_entries: + raise EvaluationError("master plan entry order or identity changed") + payload["_path"] = str(path) + payload["_sha256"] = _sha256_bytes(raw) + payload["_skill_content"] = skill["content"] + return payload + + +def _run_scorecard_validator(*args: str) -> dict[str, Any] | None: + completed = subprocess.run( + (sys.executable, str(DEFAULT_VALIDATOR_PATH), *args), + cwd=REPOSITORY_ROOT, + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise EvaluationError(f"scorecard validation failed: {detail}") + if not completed.stdout.strip().startswith("{"): + return None + value = json.loads(completed.stdout) + if not isinstance(value, dict): + raise EvaluationError("scorecard validator returned a non-object") + return value + + +def _validate_baseline_evidence() -> dict[str, Any]: + _run_scorecard_validator("check-rubric") + plan_path = _private_file(DEFAULT_BASELINE_PLAN_PATH, "baseline plan") + plan, _ = _read_json_once(plan_path, "baseline plan") + manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH) + expected_entries = [ + { + "run_id": (f"baseline__{cell['case_id']}__{cell['agent']}__r" f"{cell['repetition']}"), + "condition": "baseline", + **cell, + } + for cell in _structural_cells(manifest) + ] + expected_plan_fields = { + "protocol": PROTOCOL_ID, + "randomization_seed": RANDOMIZATION_SEED, + "repetitions": REPETITIONS, + "conditions": ["baseline"], + "manifest_sha256": baseline._sha256_file(baseline.DEFAULT_MANIFEST_PATH), + "runner_sha256": baseline._sha256_file(baseline.RUNNER_PATH), + } + for field, expected in expected_plan_fields.items(): + if plan.get(field) != expected: + raise EvaluationError(f"baseline plan {field} binding changed") + if plan.get("entries") != expected_entries: + raise EvaluationError("baseline plan identity or order changed") + + paths = sorted(DEFAULT_BASELINE_SCORES_PATH.glob("baseline__*.scorecard.json")) + if len(paths) != RUNS_PER_CONDITION: + raise EvaluationError( + f"expected {RUNS_PER_CONDITION} baseline scorecards, got {len(paths)}" + ) + expected_run_ids = {entry["run_id"] for entry in expected_entries} + records: list[dict[str, Any]] = [] + observed_run_ids: set[str] = set() + for path in paths: + summary = _run_scorecard_validator("validate", str(path)) + scorecard, _ = _read_json_once(path, "baseline scorecard") + run_id = scorecard.get("run_id") + if not isinstance(run_id, str) or run_id in observed_run_ids: + raise EvaluationError("baseline scorecard run IDs must be unique") + if summary is None or summary.get("protocol_valid") is not True: + raise EvaluationError(f"baseline scorecard {run_id!r} is not protocol-valid") + observed_run_ids.add(run_id) + records.append( + { + "run_id": run_id, + "case_id": scorecard.get("case_id"), + "score_summary": summary, + **_file_record(path), + } + ) + if observed_run_ids != expected_run_ids: + raise EvaluationError("baseline scorecard identities do not match the baseline plan") + return { + "plan": _file_record(plan_path), + "rubric": _file_record(DEFAULT_RUBRIC_PATH), + "validator": _file_record(DEFAULT_VALIDATOR_PATH), + "scorecards": records, + } + + +def _validate_shakedown_verdict(path: Path) -> dict[str, Any]: + resolved = _private_file(path, "native-delivery shakedown verdict") + verdict, _ = _read_json_once(resolved, "native-delivery shakedown verdict") + if verdict.get("kind") != SHAKEDOWN_KIND or verdict.get("protocol") != PROTOCOL_ID: + raise EvaluationError("native-delivery shakedown identity changed") + if verdict.get("runner") != _file_record(RUNNER_PATH): + raise EvaluationError("native-delivery shakedown runner binding changed") + if verdict.get("passed") is not True: + raise EvaluationError("native-delivery shakedown did not pass") + providers = verdict.get("providers") + if not isinstance(providers, dict) or set(providers) != {"codex", "claude"}: + raise EvaluationError("native-delivery shakedown must cover Codex and Claude") + skill_hash = verdict.get("synthetic_skill_sha256") + marker = verdict.get("marker") + if not isinstance(skill_hash, str) or not isinstance(marker, str): + raise EvaluationError("native-delivery shakedown is missing its canary binding") + for agent, result in providers.items(): + if not isinstance(result, dict) or result.get("passed") is not True: + raise EvaluationError(f"native-delivery shakedown failed for {agent}") + if result.get("synthetic_skill_sha256") != skill_hash: + raise EvaluationError(f"native-delivery skill bytes differed for {agent}") + if result.get("treatment_result") != marker: + raise EvaluationError(f"native-delivery marker was not observed for {agent}") + if result.get("control_result") != "NO_NATIVE_DELIVERY": + raise EvaluationError(f"native-delivery control was contaminated for {agent}") + for field in ("control_transcript", "treatment_transcript"): + record = result.get(field) + _resolve_record(record, f"{agent} {field}", private=True) + return {"verdict": verdict, "record": _file_record(resolved)} + + +def _freeze_payload( + *, + created_at: str, + plan: Mapping[str, Any], + shakedown: Mapping[str, Any], +) -> dict[str, Any]: + authorization = _private_file(DEFAULT_AUTHORIZATION_PATH, "authorization ledger") + wrapper = _private_file(DEFAULT_CLAUDE_WRAPPER_PATH, "Claude plugin wrapper") + external_pins = _private_file(DEFAULT_EXTERNAL_PINS_PATH, "external pins") + treatment_protocol = _private_file(DEFAULT_PROTOCOL_AMENDMENT_PATH, "treatment protocol") + test_path = DEFAULT_TEST_PATH.resolve(strict=True) + return { + "kind": FREEZE_KIND, + "protocol": PROTOCOL_ID, + "created_at": created_at, + "plan": _file_record(Path(str(plan["_path"]))), + "baseline": _validate_baseline_evidence(), + "frozen_inputs": { + "baseline_runner": _file_record(baseline.RUNNER_PATH), + "treatment_runner": _file_record(RUNNER_PATH), + "treatment_runner_tests": _file_record(test_path), + "protocol_document": _file_record(baseline.PROTOCOL_PATH), + "prompt_template": _file_record(baseline.PROMPT_PATH), + "case_manifest": _file_record(baseline.DEFAULT_MANIFEST_PATH), + "treatment_protocol": _file_record(treatment_protocol), + "authorization": _file_record(authorization), + "external_pins": _file_record(external_pins), + "claude_plugin_wrapper": _file_record(wrapper), + "skill": dict(plan["skill"]), + }, + "binary_bindings": {agent: _binary_binding(agent) for agent in ("codex", "claude")}, + "native_delivery_shakedown": dict(shakedown["record"]), + "measurement_policy": MEASUREMENT_POLICY, + "conditions": list(TREATMENT_CONDITIONS), + "condition_order": "all docs verdicts, then all skill verdicts", + "post_launch_scoring": ( + "No measured outcome grading until all 24 pre-score verdicts validate" + ), + "restart_policy": ( + "Quarantine and clean-restart only genuine technical interruptions; " + "never retry a completed result, refusal, timeout, or semantic failure" + ), + } + + +def create_freeze( + freeze_path: Path, + *, + plan_path: Path, + shakedown_verdict_path: Path, +) -> dict[str, Any]: + plan = validate_plan( + plan_path, + manifest_path=baseline.DEFAULT_MANIFEST_PATH, + skill_path=DEFAULT_SKILL_PATH, + external_pins_path=DEFAULT_EXTERNAL_PINS_PATH, + protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH, + authorization_path=DEFAULT_AUTHORIZATION_PATH, + claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH, + ) + shakedown = _validate_shakedown_verdict(shakedown_verdict_path) + payload = _freeze_payload( + created_at=baseline._utc_now(), + plan=plan, + shakedown=shakedown, + ) + _write_new_json(freeze_path, payload, "treatment freeze") + return payload + + +def validate_freeze( + freeze_path: Path, + *, + plan_path: Path = DEFAULT_PLAN_PATH, + shakedown_verdict_path: Path = DEFAULT_SHAKEDOWN_VERDICT_PATH, +) -> dict[str, Any]: + path = _private_file(freeze_path, "treatment freeze") + payload, raw = _read_json_once(path, "treatment freeze") + if payload.get("kind") != FREEZE_KIND or payload.get("protocol") != PROTOCOL_ID: + raise EvaluationError("treatment freeze identity changed") + created_at = payload.get("created_at") + if not isinstance(created_at, str) or not created_at: + raise EvaluationError("treatment freeze is missing created_at") + plan = validate_plan( + plan_path, + manifest_path=baseline.DEFAULT_MANIFEST_PATH, + skill_path=DEFAULT_SKILL_PATH, + external_pins_path=DEFAULT_EXTERNAL_PINS_PATH, + protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH, + authorization_path=DEFAULT_AUTHORIZATION_PATH, + claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH, + ) + shakedown = _validate_shakedown_verdict(shakedown_verdict_path) + expected = _freeze_payload( + created_at=created_at, + plan=plan, + shakedown=shakedown, + ) + if payload != expected: + raise EvaluationError("treatment freeze content changed") + payload["_path"] = str(path) + payload["_sha256"] = _sha256_bytes(raw) + payload["_plan"] = plan + return payload + + +def _claude_treatment_settings( + workspace: Path, + scratch: Path, + plugin_root: Path, +) -> dict[str, Any]: + settings = baseline._claude_settings(workspace, scratch) + environment = settings["env"] + environment.pop("HOME", None) + zdotdir = scratch / "zsh-env" + zdotdir.mkdir() + zshenv = zdotdir / ".zshenv" + zshenv.write_text(f"export HOME={shlex.quote(str(scratch / 'shell-home'))}\n") + zshenv.chmod(0o444) + environment["ZDOTDIR"] = str(zdotdir) + environment["CLAUDE_CODE_SAFE_MODE"] = "0" + permissions = settings["permissions"] + plugin_rule = "//" + str(plugin_root).lstrip("/") + zshenv_rule = "//" + str(zshenv).lstrip("/") + permissions["deny"].append(f"Edit({plugin_rule}/**)") + permissions["deny"].append(f"Edit({zshenv_rule})") + sandbox = settings["sandbox"] + sandbox["filesystem"]["denyWrite"].append(str(plugin_root)) + sandbox["filesystem"]["denyWrite"].append(str(zshenv)) + return settings + + +def _claude_treatment_command( + settings_path: Path, + plugin_root: Path, +) -> list[str]: + return [ + str(baseline.CLAUDE_BINARY), + "-p", + "--no-session-persistence", + "--no-chrome", + "--setting-sources", + "", + "--strict-mcp-config", + "--mcp-config", + '{"mcpServers":{}}', + "--settings", + str(settings_path), + "--plugin-dir", + str(plugin_root), + "--permission-mode", + "dontAsk", + "--tools", + "Read,Glob,Grep,Edit,Write,Bash,WebSearch,WebFetch,Skill", + "--model", + EXPECTED_MODELS["claude"], + "--effort", + "high", + "--max-budget-usd", + "50", + "--output-format", + "stream-json", + "--verbose", + ] + + +def _codex_treatment_command( + workspace: Path, + scratch: Path, + skills_root: Path, +) -> list[str]: + command = baseline._codex_command(workspace, scratch) + permission_index = next( + index for index, value in enumerate(command) if value.startswith("permissions.agent-eval=") + ) + permission_profile = command[permission_index] + skill_read_entry = f'{json.dumps(str(skills_root))}="read",' + command[permission_index] = permission_profile.replace( + "filesystem={", + "filesystem={" + skill_read_entry, + 1, + ) + return command + + +def _prepare_native_surface( + *, + agent: str, + condition: str, + workspace: Path, + external_run_root: Path, + scratch: Path, + environment: dict[str, str], + skill_content: bytes, + skill_name: str = SKILL_NAME, +) -> tuple[list[str], dict[str, Any], Path | None]: + skill_hash = _sha256_bytes(skill_content) + if condition not in TREATMENT_CONDITIONS: + raise EvaluationError(f"unknown treatment condition {condition!r}") + installed = condition == "skill" + if agent == "codex": + codex_home = baseline._prepare_codex_environment(external_run_root, scratch, environment) + skills_root = codex_home / "skills" + skills_root.mkdir(exist_ok=True) + if installed: + target = skills_root / skill_name / "SKILL.md" + target.parent.mkdir() + target.write_bytes(skill_content) + target.chmod(0o444) + surface = { + "kind": "isolated-codex-home-skill-v1", + "skill_installed": installed, + "semantic_file_count": 1 if installed else 0, + "skill_sha256": skill_hash if installed else None, + } + return _codex_treatment_command(workspace, scratch, skills_root), surface, None + + if agent != "claude": + raise EvaluationError(f"unknown agent {agent!r}") + plugin_source = DEFAULT_CLAUDE_WRAPPER_PATH.parents[1] + plugin_root = scratch / "claude-plugin" + shutil.copytree(plugin_source, plugin_root) + if installed: + target = plugin_root / "skills" / skill_name / "SKILL.md" + target.parent.mkdir(parents=True) + target.write_bytes(skill_content) + target.chmod(0o444) + surface = { + "kind": "isolated-claude-plugin-skill-v1", + "skill_installed": installed, + "semantic_file_count": 1 if installed else 0, + "skill_sha256": skill_hash if installed else None, + "wrapper_manifest_sha256": baseline._sha256_file( + plugin_root / ".claude-plugin" / "plugin.json" + ), + } + return [], surface, plugin_root + + +def _audit_native_surface( + *, + agent: str, + condition: str, + external_run_root: Path, + scratch: Path, + skill_content: bytes, + skill_name: str = SKILL_NAME, +) -> dict[str, Any]: + if agent == "codex": + path = external_run_root / "codex-home" / "skills" / skill_name / "SKILL.md" + elif agent == "claude": + path = scratch / "claude-plugin" / "skills" / skill_name / "SKILL.md" + else: + raise EvaluationError(f"unknown agent {agent!r}") + expected = condition == "skill" + present = path.is_file() and not path.is_symlink() + actual_hash = baseline._sha256_file(path) if present else None + expected_hash = _sha256_bytes(skill_content) if expected else None + return { + "expected_present": expected, + "present": present, + "path": str(path), + "expected_sha256": expected_hash, + "final_sha256": actual_hash, + "mutated": present != expected or actual_hash != expected_hash, + } + + +def _transcript_terminal_state(path: Path, agent: str) -> str: + completed = False + failed = False + try: + with path.open(encoding="utf-8") as handle: + for line in handle: + try: + event = json.loads(line) + except json.JSONDecodeError: + failed = True + continue + if not isinstance(event, dict): + continue + if agent == "codex": + if event.get("type") == "turn.completed": + completed = True + if event.get("type") in {"turn.failed", "error"}: + failed = True + else: + if event.get("type") == "result": + terminal_reason = event.get("terminal_reason") + subtype = event.get("subtype") + if terminal_reason == "completed" and subtype == "success": + completed = True + else: + failed = True + except (OSError, UnicodeDecodeError): + return "unreadable" + if completed and not failed: + return "completed" + if completed: + return "completed-with-error-events" + if failed: + return "failed" + return "incomplete" + + +def _claude_catalog_audit( + path: Path, + *, + condition: str, + skill_name: str = SKILL_NAME, +) -> dict[str, Any]: + init: dict[str, Any] | None = None + with path.open(encoding="utf-8") as handle: + for line in handle: + event = json.loads(line) + if ( + isinstance(event, dict) + and event.get("type") == "system" + and event.get("subtype") == "init" + ): + init = event + break + if init is None: + return {"valid": False, "reason": "missing system init event"} + plugins = init.get("plugins") + plugin_names = ( + [item.get("name") for item in plugins if isinstance(item, dict)] + if isinstance(plugins, list) + else [] + ) + raw_skills = init.get("skills") + skills: list[Any] = raw_skills if isinstance(raw_skills, list) else [] + measured_name = f"agent-qualification-treatment:{skill_name}" + measured_present = measured_name in skills + expected_present = condition == "skill" + errors: list[str] = [] + if plugin_names != ["agent-qualification-treatment"]: + errors.append("unexpected plugin catalog") + if init.get("mcp_servers") != []: + errors.append("MCP catalog was not empty") + if measured_present is not expected_present: + errors.append("measured skill catalog presence changed") + required_tools = { + "Bash", + "Edit", + "Glob", + "Grep", + "Read", + "Skill", + "WebFetch", + "WebSearch", + "Write", + } + if set(init.get("tools", [])) != required_tools: + errors.append("tool catalog changed") + return { + "valid": not errors, + "errors": errors, + "plugin_names": plugin_names, + "mcp_servers": init.get("mcp_servers"), + "measured_skill": measured_name, + "measured_skill_expected": expected_present, + "measured_skill_present": measured_present, + "model": init.get("model"), + } + + +def _entry_by_run_id(plan: Mapping[str, Any], run_id: str) -> dict[str, Any]: + entries = plan.get("entries") + if not isinstance(entries, list): + raise EvaluationError("validated plan lost its entries") + matches = [entry for entry in entries if entry.get("run_id") == run_id] + if len(matches) != 1 or not isinstance(matches[0], dict): + raise EvaluationError(f"run {run_id!r} is not one frozen plan entry") + return matches[0] + + +def _run_root(run_id: str) -> Path: + return DEFAULT_RUNS_PATH.resolve() / run_id + + +def _validate_prior_verdicts( + plan: Mapping[str, Any], + freeze: Mapping[str, Any], + ordinal: int, +) -> None: + entries = plan.get("entries") + if not isinstance(entries, list): + raise EvaluationError("validated plan lost its entries") + for entry in entries[: ordinal - 1]: + validate_run(str(entry["run_id"]), plan=plan, freeze=freeze) + + +def _expected_metadata_bindings( + entry: Mapping[str, Any], + *, + freeze: Mapping[str, Any], + plan: Mapping[str, Any], + case: Mapping[str, Any], +) -> dict[str, Any]: + agent = str(entry["agent"]) + binary_binding = freeze["binary_bindings"][agent] + return { + "protocol": PROTOCOL_ID, + "run_id": entry["run_id"], + "evidence_id": entry["evidence_id"], + "ordinal": entry["ordinal"], + "condition": entry["condition"], + "case_id": entry["case_id"], + "agent": agent, + "repetition": entry["repetition"], + "manifest_sha256": baseline._sha256_file(baseline.DEFAULT_MANIFEST_PATH), + "gold_manifest_sha256": str(case["gold_sha256"]), + "baseline_runner_sha256": baseline._sha256_file(baseline.RUNNER_PATH), + "treatment_runner_sha256": baseline._sha256_file(RUNNER_PATH), + "treatment_freeze_sha256": freeze["_sha256"], + "master_plan_sha256": plan["_sha256"], + "protocol_sha256": baseline._sha256_file(baseline.PROTOCOL_PATH), + "prompt_template_sha256": baseline._sha256_file(baseline.PROMPT_PATH), + "docs_treatment_sha256": _sha256_bytes(baseline.DOCS_TREATMENT.encode()), + "rendered_prompt_sha256": entry["rendered_prompt_sha256"], + "snapshot_sha256": str(case["snapshot_sha256"]), + "requested_model": EXPECTED_MODELS[agent], + "binary_sha256": binary_binding["sha256"], + "cli_version": binary_binding["cli_version"], + "timeout_seconds": baseline.DEFAULT_TIMEOUT_SECONDS, + "assigned_payload_is_intervention": False, + "post_launch_intervention_class": "I0", + } + + +def _write_pre_score_verdict( + run_root: Path, + *, + valid: bool, + errors: Sequence[str], + metadata: Mapping[str, Any], +) -> dict[str, Any]: + verdict = { + "kind": VERDICT_KIND, + "protocol": PROTOCOL_ID, + "run_id": metadata.get("run_id"), + "valid": valid, + "errors": list(errors), + "terminal_state": metadata.get("terminal_state"), + "timed_out": metadata.get("timed_out"), + "exit_code": metadata.get("exit_code"), + "metadata_sha256": baseline._sha256_file(run_root / "metadata.json"), + "transcript_sha256": metadata.get("transcript_sha256"), + "created_at": baseline._utc_now(), + } + path = run_root / "pre-score-verdict.json" + if path.exists(): + raise EvaluationError(f"refusing to overwrite run verdict {path}") + path.write_bytes(_json_bytes(verdict)) + return verdict + + +def validate_run( + run_id: str, + *, + plan: Mapping[str, Any] | None = None, + freeze: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + if freeze is None: + freeze = validate_freeze(DEFAULT_FREEZE_PATH) + if plan is None: + raw_plan = freeze.get("_plan") + if not isinstance(raw_plan, dict): + raise EvaluationError("validated freeze lost its master plan") + plan = raw_plan + entry = _entry_by_run_id(plan, run_id) + run_root = _run_root(run_id) + metadata_path = _private_file(run_root / "metadata.json", "run metadata") + metadata, _ = _read_json_once(metadata_path, "run metadata") + manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH) + case = baseline._case_index(manifest)[str(entry["case_id"])] + errors: list[str] = [] + for field, expected in _expected_metadata_bindings( + entry, freeze=freeze, plan=plan, case=case + ).items(): + if metadata.get(field) != expected: + errors.append(f"metadata {field} binding changed") + agent = str(entry["agent"]) + transcript_path = run_root / f"{agent}.jsonl" + stderr_path = run_root / f"{agent}.stderr" + for artifact in (transcript_path, stderr_path, run_root / "prompt.txt"): + if not artifact.is_file() or artifact.is_symlink(): + errors.append(f"missing required run artifact {artifact.name}") + if transcript_path.is_file(): + actual_transcript_hash = baseline._sha256_file(transcript_path) + if metadata.get("transcript_sha256") != actual_transcript_hash: + errors.append("transcript hash changed") + if stderr_path.is_file(): + actual_stderr_hash = baseline._sha256_file(stderr_path) + if metadata.get("stderr_sha256") != actual_stderr_hash: + errors.append("stderr hash changed") + prompt_path = run_root / "prompt.txt" + if prompt_path.is_file() and baseline._sha256_file(prompt_path) != entry.get( + "rendered_prompt_sha256" + ): + errors.append("rendered prompt artifact changed") + if metadata.get("model_identifier") != EXPECTED_MODELS[agent]: + errors.append("reported model identifier changed") + expected_skill = entry["condition"] == "skill" + if metadata.get("skill_injected") is not expected_skill: + errors.append("skill injection flag changed") + if metadata.get("frozen_skill_sha256") != plan["skill"]["sha256"]: + errors.append("frozen skill metadata binding changed") + surface = metadata.get("native_surface") + if not isinstance(surface, dict): + errors.append("native skill surface audit is missing") + else: + expected_kind = ( + "isolated-codex-home-skill-v1" + if agent == "codex" + else "isolated-claude-plugin-skill-v1" + ) + if surface.get("kind") != expected_kind: + errors.append("native skill surface kind changed") + if surface.get("skill_installed") is not expected_skill: + errors.append("native skill surface condition changed") + if surface.get("semantic_file_count") != (1 if expected_skill else 0): + errors.append("native skill semantic file count changed") + expected_skill_hash = plan["skill"]["sha256"] if expected_skill else None + if surface.get("skill_sha256") != expected_skill_hash: + errors.append("native skill surface hash changed") + final_audit = surface.get("final_audit") + if not isinstance(final_audit, dict) or final_audit.get("mutated") is not False: + errors.append("native skill surface final audit failed") + if agent == "claude": + catalog_audit = surface.get("provider_catalog_audit") + if not isinstance(catalog_audit, dict) or catalog_audit.get("valid") is not True: + errors.append("Claude provider catalog audit failed") + terminal_state = metadata.get("terminal_state") + timed_out = metadata.get("timed_out") + if timed_out is not True and terminal_state != "completed": + errors.append("provider subprocess did not produce a completed terminal event") + fixtures = metadata.get("fixtures") + if not isinstance(fixtures, dict): + errors.append("fixture audit is missing") + else: + for name, fixture in fixtures.items(): + if not isinstance(fixture, dict): + errors.append(f"fixture {name} audit is invalid") + continue + if fixture.get("kind") == "none": + continue + if fixture.get("mutated") is not False: + errors.append(f"fixture {name} mutated") + if fixture.get("final_tree_sha256") != fixture.get("initial_tree_sha256"): + errors.append(f"fixture {name} final hash changed") + verdict_path = run_root / "pre-score-verdict.json" + if not verdict_path.is_file(): + raise EvaluationError(f"run {run_id!r} is missing its pre-score verdict") + verdict, _ = _read_json_once(verdict_path, "pre-score verdict") + if verdict.get("kind") != VERDICT_KIND or verdict.get("run_id") != run_id: + errors.append("pre-score verdict identity changed") + if verdict.get("metadata_sha256") != baseline._sha256_file(metadata_path): + errors.append("pre-score verdict metadata hash changed") + if verdict.get("transcript_sha256") != metadata.get("transcript_sha256"): + errors.append("pre-score verdict transcript hash changed") + if verdict.get("valid") is not True or verdict.get("errors") != []: + errors.append("pre-score verdict did not record a valid run") + if errors: + raise EvaluationError("; ".join(errors)) + return verdict + + +def execute_run( + run_id: str, + *, + timeout_seconds: int = baseline.DEFAULT_TIMEOUT_SECONDS, +) -> dict[str, Any]: + if timeout_seconds != baseline.DEFAULT_TIMEOUT_SECONDS: + raise EvaluationError( + "official treatment timeout must remain " f"{baseline.DEFAULT_TIMEOUT_SECONDS} seconds" + ) + freeze = validate_freeze(DEFAULT_FREEZE_PATH) + raw_plan = freeze.get("_plan") + if not isinstance(raw_plan, dict): + raise EvaluationError("validated freeze lost its master plan") + plan = raw_plan + entry = _entry_by_run_id(plan, run_id) + _validate_prior_verdicts(plan, freeze, int(entry["ordinal"])) + + manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH) + case = baseline._case_index(manifest)[str(entry["case_id"])] + if case.get("status") != "ready": + raise EvaluationError(f"case {entry['case_id']!r} is not ready") + run_root = _run_root(run_id) + if run_root.exists() or run_root.is_symlink(): + raise EvaluationError(f"refusing to overwrite existing run {run_root}") + + agent = str(entry["agent"]) + condition = str(entry["condition"]) + source = (REPOSITORY_ROOT / str(case["source"])).resolve(strict=True) + work_root = baseline.CODEX_WORK_ROOT if agent == "codex" else baseline.CLAUDE_WORK_ROOT + external_run_root = work_root / run_id + if external_run_root.exists() or external_run_root.is_symlink(): + raise EvaluationError(f"refusing to reuse external workspace {external_run_root}") + workspace = external_run_root / "workspace" + scratch = external_run_root / "scratch" + transcript_path = run_root / f"{agent}.jsonl" + stderr_path = run_root / f"{agent}.stderr" + + run_root.mkdir(parents=True) + scratch.mkdir(parents=True) + baseline._initialize_snapshot(source, workspace) + prompt = render_prompt(case) + if _sha256_bytes(prompt.encode()) != entry["rendered_prompt_sha256"]: + raise EvaluationError("rendered treatment prompt changed before launch") + (run_root / "prompt.txt").write_text(prompt) + environment = baseline._clean_agent_environment(scratch) + fixtures = baseline._prepare_case_fixtures(case, scratch) + skill_content = plan.get("_skill_content") + if not isinstance(skill_content, bytes): + raise EvaluationError("validated plan lost the frozen skill bytes") + command, native_surface, plugin_root = _prepare_native_surface( + agent=agent, + condition=condition, + workspace=workspace, + external_run_root=external_run_root, + scratch=scratch, + environment=environment, + skill_content=skill_content, + ) + settings_path: Path | None = None + if agent == "claude": + if plugin_root is None: + raise EvaluationError("Claude native plugin root was not prepared") + settings_path = run_root / "claude-settings.json" + baseline._write_json( + settings_path, + _claude_treatment_settings(workspace, scratch, plugin_root), + ) + command = _claude_treatment_command(settings_path, plugin_root) + + binary = baseline.CODEX_BINARY if agent == "codex" else baseline.CLAUDE_BINARY + cli_version = baseline._binary_version(binary, environment) + started_at = baseline._utc_now() + bindings = _expected_metadata_bindings(entry, freeze=freeze, plan=plan, case=case) + metadata: dict[str, Any] = { + **bindings, + "started_at": started_at, + "timeout_seconds": timeout_seconds, + "command": command, + "prompt_transport": "stdin", + "cli_version": cli_version, + "binary_sha256": baseline._sha256_file(binary), + "removed_environment_names": sorted(baseline.SECRET_ENVIRONMENT_NAMES), + "fixtures": fixtures, + "native_surface": native_surface, + "skill_injected": condition == "skill", + "frozen_skill_sha256": plan["skill"]["sha256"], + "treatment_payload_transport": "native local skill mechanism", + } + baseline._write_json(run_root / "metadata.json", metadata) + + exit_code, timed_out, elapsed_seconds = baseline._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript_path, + stderr_path=stderr_path, + timeout_seconds=timeout_seconds, + ) + native_surface["final_audit"] = _audit_native_surface( + agent=agent, + condition=condition, + external_run_root=external_run_root, + scratch=scratch, + skill_content=skill_content, + ) + baseline._write_workspace_evidence(workspace, run_root) + baseline._record_final_fixture_state(fixtures, scratch) + shutil.copytree( + workspace, + run_root / "workspace-final", + symlinks=True, + ignore=shutil.ignore_patterns( + ".git", "bin", "obj", "target", "node_modules", ".eval-cache" + ), + ) + terminal_state = _transcript_terminal_state(transcript_path, agent) + if agent == "claude": + native_surface["provider_catalog_audit"] = _claude_catalog_audit( + transcript_path, + condition=condition, + ) + metadata.update( + { + "completed_at": baseline._utc_now(), + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed_seconds, + "terminal_state": terminal_state, + "transcript_sha256": baseline._sha256_file(transcript_path), + "stderr_sha256": baseline._sha256_file(stderr_path), + "fixtures": fixtures, + } + ) + model_identifier, model_identifier_source = baseline._reported_model(transcript_path, agent) + metadata["model_identifier"] = model_identifier + metadata["model_identifier_source"] = model_identifier_source + baseline._write_json(run_root / "metadata.json", metadata) + + pre_errors: list[str] = [] + if model_identifier != EXPECTED_MODELS[agent]: + pre_errors.append(f"reported model {model_identifier!r} != {EXPECTED_MODELS[agent]!r}") + if timed_out is not True and terminal_state != "completed": + pre_errors.append(f"provider terminal state was {terminal_state!r}") + if native_surface["final_audit"]["mutated"] is not False: + pre_errors.append("native skill surface mutated") + if agent == "claude" and native_surface["provider_catalog_audit"]["valid"] is not True: + pre_errors.append("Claude provider catalog isolation failed") + for name, fixture in fixtures.items(): + if fixture.get("kind") != "none" and fixture.get("mutated") is not False: + pre_errors.append(f"fixture {name} mutated") + verdict = _write_pre_score_verdict( + run_root, + valid=not pre_errors, + errors=pre_errors, + metadata=metadata, + ) + if pre_errors: + raise EvaluationError(f"run {run_id!r} is technically invalid: " + "; ".join(pre_errors)) + validate_run(run_id, plan=plan, freeze=freeze) + shutil.rmtree(external_run_root) + return {"metadata": metadata, "verdict": verdict} + + +def _transcript_result_text(path: Path, agent: str) -> str: + result = "" + with path.open(encoding="utf-8") as handle: + for line in handle: + event = json.loads(line) + if not isinstance(event, dict): + continue + if agent == "codex" and event.get("type") == "item.completed": + item = event.get("item") + if isinstance(item, dict) and item.get("type") == "agent_message": + text = item.get("text") + if isinstance(text, str): + result = text + if agent == "claude" and event.get("type") == "result": + text = event.get("result") + if isinstance(text, str): + result = text + return result.strip() + + +def run_native_delivery_shakedown( + *, + output_path: Path = DEFAULT_SHAKEDOWN_VERDICT_PATH, +) -> dict[str, Any]: + prior_root = PRIVATE_ROOT / "shakedowns" / "native-skill-delivery-20260729" + skill_source = _private_file( + prior_root / "skill-source" / "SKILL.md", "synthetic shakedown skill" + ) + skill_content = skill_source.read_bytes() + skill_hash = _sha256_bytes(skill_content) + marker = "NATIVE_SKILL_PROBE_20260729_8C41D6" + if marker.encode() not in skill_content: + raise EvaluationError("synthetic shakedown skill lost its marker") + verified_root = PRIVATE_ROOT / "shakedowns" / "native-skill-delivery-verified-20260729" + if verified_root.exists() or verified_root.is_symlink(): + raise EvaluationError(f"refusing to reuse shakedown root {verified_root}") + verified_root.mkdir(parents=True) + prompt = ( + "Run the native-delivery-probe shakedown now. If its native skill is " + "available, use it. If unavailable, reply exactly " + "NO_NATIVE_DELIVERY. Do not search for skill files or inspect any path " + "not supplied by the native skill catalog." + ) + providers: dict[str, Any] = {} + for agent in ("codex", "claude"): + agent_result: dict[str, Any] = { + "synthetic_skill_sha256": skill_hash, + } + for condition, label in (("docs", "control"), ("skill", "treatment")): + root = verified_root / f"{agent}-{label}" + workspace = root / "workspace" + scratch = root / "scratch" + workspace.mkdir(parents=True) + scratch.mkdir(parents=True) + (workspace / "README.md").write_text("Synthetic native skill delivery probe.\n") + environment = baseline._clean_agent_environment(scratch) + command, surface, plugin_root = _prepare_native_surface( + agent=agent, + condition=condition, + workspace=workspace, + external_run_root=root, + scratch=scratch, + environment=environment, + skill_content=skill_content, + skill_name="native-delivery-probe", + ) + if agent == "claude": + if plugin_root is None: + raise EvaluationError("Claude shakedown plugin was not prepared") + settings_path = root / "claude-settings.json" + baseline._write_json( + settings_path, + _claude_treatment_settings(workspace, scratch, plugin_root), + ) + command = _claude_treatment_command(settings_path, plugin_root) + transcript = root / f"{agent}.jsonl" + stderr = root / f"{agent}.stderr" + exit_code, timed_out, elapsed = baseline._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript, + stderr_path=stderr, + timeout_seconds=600, + ) + surface["final_audit"] = _audit_native_surface( + agent=agent, + condition=condition, + external_run_root=root, + scratch=scratch, + skill_content=skill_content, + skill_name="native-delivery-probe", + ) + if agent == "claude": + surface["provider_catalog_audit"] = _claude_catalog_audit( + transcript, + condition=condition, + skill_name="native-delivery-probe", + ) + terminal_state = _transcript_terminal_state(transcript, agent) + result = _transcript_result_text(transcript, agent) + model, model_source = baseline._reported_model(transcript, agent) + expected_result = marker if condition == "skill" else "NO_NATIVE_DELIVERY" + if exit_code != 0 or timed_out or terminal_state != "completed": + raise EvaluationError(f"{agent} {label} shakedown did not complete cleanly") + if surface["final_audit"]["mutated"] is not False: + raise EvaluationError(f"{agent} {label} native skill surface mutated") + if agent == "claude" and surface["provider_catalog_audit"]["valid"] is not True: + raise EvaluationError(f"{agent} {label} provider catalog was not isolated") + if result != expected_result: + raise EvaluationError( + f"{agent} {label} shakedown returned {result!r}, " + f"expected {expected_result!r}" + ) + if model != EXPECTED_MODELS[agent]: + raise EvaluationError(f"{agent} shakedown model changed: {model!r}") + agent_result[f"{label}_result"] = result + agent_result[f"{label}_transcript"] = _file_record(transcript) + agent_result[f"{label}_stderr"] = _file_record(stderr) + agent_result[f"{label}_surface"] = surface + agent_result[f"{label}_elapsed_seconds"] = elapsed + agent_result[f"{label}_model"] = model + agent_result[f"{label}_model_source"] = model_source + agent_result["passed"] = True + providers[agent] = agent_result + verdict = { + "kind": SHAKEDOWN_KIND, + "protocol": PROTOCOL_ID, + "created_at": baseline._utc_now(), + "passed": True, + "runner": _file_record(RUNNER_PATH), + "marker": marker, + "synthetic_skill": _file_record(skill_source), + "synthetic_skill_sha256": skill_hash, + "prompt_sha256": _sha256_bytes(prompt.encode()), + "providers": providers, + } + _write_new_json(output_path, verdict, "native-delivery shakedown verdict") + return verdict + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("shakedown", help="verify native skill delivery on a synthetic case") + + plan = subparsers.add_parser("plan", help="create the immutable 24-run plan") + plan.add_argument("--output", type=Path, default=DEFAULT_PLAN_PATH) + + freeze = subparsers.add_parser("freeze", help="freeze all treatment inputs") + freeze.add_argument("--output", type=Path, default=DEFAULT_FREEZE_PATH) + + subparsers.add_parser("validate", help="validate the complete treatment freeze") + + run = subparsers.add_parser("run", help="execute the next frozen plan entry") + run.add_argument("--run-id", required=True) + run.add_argument("--timeout-seconds", type=int, default=baseline.DEFAULT_TIMEOUT_SECONDS) + + validate_run_parser = subparsers.add_parser( + "validate-run", help="validate one completed pre-score run" + ) + validate_run_parser.add_argument("--run-id", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + try: + if args.command == "shakedown": + payload = run_native_delivery_shakedown() + elif args.command == "plan": + payload = create_plan( + baseline.DEFAULT_MANIFEST_PATH, + skill_path=DEFAULT_SKILL_PATH, + external_pins_path=DEFAULT_EXTERNAL_PINS_PATH, + protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH, + authorization_path=DEFAULT_AUTHORIZATION_PATH, + claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH, + output_path=args.output, + ) + elif args.command == "freeze": + payload = create_freeze( + args.output, + plan_path=DEFAULT_PLAN_PATH, + shakedown_verdict_path=DEFAULT_SHAKEDOWN_VERDICT_PATH, + ) + elif args.command == "validate": + payload = validate_freeze(DEFAULT_FREEZE_PATH) + elif args.command == "run": + payload = execute_run(args.run_id, timeout_seconds=args.timeout_seconds) + elif args.command == "validate-run": + payload = validate_run(args.run_id) + else: # pragma: no cover - argparse enforces the command. + raise EvaluationError(f"unknown command {args.command!r}") + except (EvaluationError, OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + printable = { + key: value + for key, value in payload.items() + if not key.startswith("_") and not isinstance(value, bytes) + } + print(json.dumps(printable, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 966bc6f..6d6eae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,9 @@ addopts = "-q" [tool.black] line-length = 100 target-version = ["py310"] +# This runner is hash-bound by the frozen v2 cohort and cannot be reformatted +# without invalidating the preregistration. +extend-exclude = "^/experiments/agent_qualification_generalization.py$" [tool.mypy] python_version = "3.10" @@ -103,6 +106,7 @@ check_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true warn_unused_configs = true +exclude = "^experiments/private/" [tool.coverage.run] source = ["tracebook"] diff --git a/tests/test_agent_qualification_experiment.py b/tests/test_agent_qualification_experiment.py new file mode 100644 index 0000000..97b077a --- /dev/null +++ b/tests/test_agent_qualification_experiment.py @@ -0,0 +1,333 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from experiments import agent_qualification + + +def _file_hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _manifest(tmp_path: Path, *, status: str = "ready") -> Path: + source = tmp_path / "source" + source.mkdir(parents=True) + (source / "engine.txt").write_text("price-time\n") + gold = tmp_path / "gold.json" + gold.write_text('{"terminal_class":"conformant"}\n') + manifest = { + "protocol": agent_qualification.PROTOCOL_ID, + "protocol_sha256": _file_hash(agent_qualification.PROTOCOL_PATH), + "prompt_sha256": _file_hash(agent_qualification.PROMPT_PATH), + "cases": [ + { + "id": "c1", + "source": str(source), + "snapshot_id": "snapshot-c1", + "revision": "revision-c1", + "commands": "test-engine", + "declared_claim": "FIFO matching", + "excluded_scope": "latency", + "snapshot_sha256": agent_qualification._snapshot_digest(source), + "gold_manifest": str(gold), + "gold_sha256": _file_hash(gold), + "dependency_cache": {"kind": "none"}, + "native_test_tool": {"kind": "none"}, + "status": status, + } + ], + } + path = tmp_path / "cases.json" + path.write_text(json.dumps(manifest)) + return path + + +def _add_frozen_fixture( + manifest_path: Path, + fixture_root: Path, + *, + field_name: str, + kind: str, +) -> None: + source = fixture_root / "source" + source.mkdir(parents=True) + (source / "package.bin").write_bytes(b"frozen dependency") + case = json.loads(manifest_path.read_text())["cases"][0] + inventory = agent_qualification._tree_inventory(source) + tree_hash = agent_qualification._snapshot_digest(source) + fixture_manifest = { + "kind": kind, + "case_id": case["id"], + "snapshot_sha256": case["snapshot_sha256"], + "tree_sha256": tree_hash, + "file_count": len(inventory), + "total_bytes": sum(item["bytes"] for item in inventory), + "files": inventory, + } + fixture_manifest_path = fixture_root / "manifest.json" + fixture_manifest_path.write_text(json.dumps(fixture_manifest)) + payload = json.loads(manifest_path.read_text()) + payload["cases"][0][field_name] = { + "kind": kind, + "source": str(source), + "manifest": str(fixture_manifest_path), + "manifest_sha256": _file_hash(fixture_manifest_path), + "tree_sha256": tree_hash, + } + manifest_path.write_text(json.dumps(payload)) + + +def test_validate_manifest_rejects_snapshot_drift(tmp_path): + manifest_path = _manifest(tmp_path) + manifest = agent_qualification.validate_manifest(manifest_path) + assert manifest["protocol"] == agent_qualification.PROTOCOL_ID + + source = Path(manifest["cases"][0]["source"]) + (source / "engine.txt").write_text("changed\n") + + with pytest.raises(agent_qualification.EvaluationError, match="snapshot changed"): + agent_qualification.validate_manifest(manifest_path) + + +def test_validate_manifest_rejects_gold_drift(tmp_path): + manifest_path = _manifest(tmp_path) + manifest = agent_qualification.validate_manifest(manifest_path) + + Path(manifest["cases"][0]["gold_manifest"]).write_text( + '{"terminal_class":"candidate-defect"}\n' + ) + + with pytest.raises(agent_qualification.EvaluationError, match="gold manifest changed"): + agent_qualification.validate_manifest(manifest_path) + + +def test_frozen_cache_copy_is_verified_and_isolated(tmp_path, monkeypatch): + manifest_path = _manifest(tmp_path) + _add_frozen_fixture( + manifest_path, + tmp_path / "fixtures" / "nuget", + field_name="dependency_cache", + kind="nuget-global-packages-v1", + ) + monkeypatch.setattr(agent_qualification, "PRIVATE_ROOT", tmp_path) + + manifest = agent_qualification.validate_manifest(manifest_path) + case = manifest["cases"][0] + scratch = tmp_path / "scratch" + agent_qualification._clean_agent_environment(scratch) + prepared = agent_qualification._prepare_case_fixtures(case, scratch) + + assert ( + prepared["dependency_cache"]["initial_tree_sha256"] + == case["dependency_cache"]["tree_sha256"] + ) + copied = scratch / "nuget" / "package.bin" + copied.write_bytes(b"mutated run copy") + source = Path(case["dependency_cache"]["source"]) / "package.bin" + assert source.read_bytes() == b"frozen dependency" + agent_qualification._record_final_fixture_state(prepared, scratch) + assert prepared["dependency_cache"]["mutated"] is True + + +def test_frozen_cache_rejects_tree_drift(tmp_path, monkeypatch): + manifest_path = _manifest(tmp_path) + fixture_root = tmp_path / "fixtures" / "nuget" + _add_frozen_fixture( + manifest_path, + fixture_root, + field_name="dependency_cache", + kind="nuget-global-packages-v1", + ) + monkeypatch.setattr(agent_qualification, "PRIVATE_ROOT", tmp_path) + (fixture_root / "source" / "package.bin").write_bytes(b"drift") + + with pytest.raises(agent_qualification.EvaluationError, match="manifest binding"): + agent_qualification.validate_manifest(manifest_path) + + +def test_render_prompt_keeps_baseline_blind(): + case = { + "id": "c1", + "snapshot_id": "snapshot-c1", + "revision": "revision-c1", + "commands": "test-engine", + "declared_claim": "FIFO matching", + "excluded_scope": "latency", + } + + baseline = agent_qualification.render_prompt(case, "baseline") + docs = agent_qualification.render_prompt(case, "docs") + + assert "tracebook" not in baseline.lower() + assert "tracebook-conformance==0.6.0" in docs + assert "{{" not in baseline + assert "snapshot-c1" in baseline + + +def test_plan_is_seeded_and_excludes_pending_cases(tmp_path): + manifest_path = _manifest(tmp_path) + pending_path = _manifest(tmp_path / "pending", status="pending-gold") + first_path = tmp_path / "first.json" + second_path = tmp_path / "second.json" + pending_plan_path = tmp_path / "pending-plan.json" + + first = agent_qualification.create_plan( + manifest_path, first_path, repetitions=3, conditions=("baseline",) + ) + second = agent_qualification.create_plan( + manifest_path, second_path, repetitions=3, conditions=("baseline",) + ) + pending = agent_qualification.create_plan( + pending_path, + pending_plan_path, + repetitions=3, + conditions=("baseline",), + ) + + assert first["entries"] == second["entries"] + assert len(first["entries"]) == 6 + assert {entry["agent"] for entry in first["entries"]} == {"codex", "claude"} + assert pending["entries"] == [] + + +def test_skill_condition_is_locked_before_baseline(tmp_path): + manifest_path = _manifest(tmp_path) + + with pytest.raises(agent_qualification.EvaluationError, match="remain locked"): + agent_qualification.execute_run( + manifest_path, + case_id="c1", + agent="codex", + repetition=1, + condition="skill", + runs_path=tmp_path / "runs", + timeout_seconds=1, + ) + + +def test_claude_permissions_delegate_paths_to_fail_closed_sandbox(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + workspace.mkdir() + scratch.mkdir() + + settings = agent_qualification._claude_settings(workspace, scratch) + + assert settings["permissions"]["allow"] == [ + f"Read(/{workspace}/**)", + f"Edit(/{workspace}/**)", + "Bash", + "WebSearch", + "WebFetch", + ] + assert f"Read(/{Path.home()}/**)" in settings["permissions"]["deny"] + assert f"Edit(/{workspace}/.git/**)" in settings["permissions"]["deny"] + assert str(workspace) in settings["sandbox"]["filesystem"]["allowWrite"] + assert str(workspace / ".git") in settings["sandbox"]["filesystem"]["denyWrite"] + assert str(Path.home()) in settings["sandbox"]["filesystem"]["denyRead"] + assert str(Path("/usr/local/Cellar")) in settings["sandbox"]["filesystem"]["allowRead"] + assert settings["env"]["PATH"].startswith( + str(Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent) + ) + assert settings["env"]["DOTNET_ROOT"] == str( + Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent.parent / "libexec" + ) + assert settings["env"]["NuGetAudit"] == "false" + + +def test_dotnet_environment_preseeds_nuget_migration_state(tmp_path): + environment = agent_qualification._dotnet_environment(tmp_path) + + xdg_data_home = tmp_path / "xdg-data" + assert environment["XDG_DATA_HOME"] == str(xdg_data_home) + assert (xdg_data_home / "NuGet" / "Migrations" / "1").is_file() + + +def test_codex_command_uses_pure_permission_profile(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + workspace.mkdir() + scratch.mkdir() + + command = agent_qualification._codex_command(workspace, scratch) + permission_override = next( + item for item in command if item.startswith("permissions.agent-eval=") + ) + + assert command[-1] == "-" + assert "--sandbox" not in command + assert "workspace-write" not in command + assert command.index("exec") > command.index("--disable") + assert 'default_permissions="agent-eval"' in command + assert '":root"="deny"' in permission_override + assert '":workspace_roots"={"."="write",".git"="read"}' in permission_override + assert f'{scratch}"="write"' in permission_override + assert '"/usr/local/bin/dotnet"="read"' in permission_override + assert '"/usr/local/share/dotnet"="read"' in permission_override + assert '"/usr/local/opt"="read"' in permission_override + assert '"/usr/local/Cellar"="read"' in permission_override + assert 'mode="limited"' in permission_override + assert "allow_upstream_proxy=false" in permission_override + assert "allow_local_binding=true" not in permission_override + assert '"pypi.org"="allow"' in permission_override + assert "nuget.org" not in permission_override + assert 'shell_environment_policy.inherit="core"' in command + assert any(item.startswith("shell_environment_policy.set=") for item in command) + assert "--search" in command + shell_environment = agent_qualification._codex_shell_environment(scratch) + assert shell_environment["PATH"].startswith( + str(Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent) + ) + assert shell_environment["DOTNET_ROOT"] == str( + Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent.parent / "libexec" + ) + + +def test_clean_environment_removes_secret_shaped_names(tmp_path, monkeypatch): + monkeypatch.setenv("AGENT_EVAL_SAFE_VALUE", "safe") + monkeypatch.setenv("UNLISTED_PRIVATE_TOKEN", "secret") + monkeypatch.setenv("SSH_AUTH_SOCK", "/private/tmp/agent.sock") + + environment = agent_qualification._clean_agent_environment(tmp_path) + + assert environment["AGENT_EVAL_SAFE_VALUE"] == "safe" + assert "UNLISTED_PRIVATE_TOKEN" not in environment + assert "SSH_AUTH_SOCK" not in environment + + +def test_codex_home_is_outside_agent_writable_scratch(tmp_path, monkeypatch): + external_root = tmp_path / "run" + scratch = external_root / "scratch" + external_root.mkdir() + scratch.mkdir() + auth = tmp_path / "auth.json" + auth.write_text("{}\n") + monkeypatch.setattr(agent_qualification, "CODEX_AUTH_PATH", auth) + environment: dict[str, str] = {} + + codex_home = agent_qualification._prepare_codex_environment(external_root, scratch, environment) + + assert codex_home == external_root / "codex-home" + assert scratch not in codex_home.parents + assert (codex_home / "auth.json").is_symlink() + assert Path(environment["HOME"]) == scratch / "shell-home" + assert Path(environment["CODEX_HOME"]) == codex_home + assert Path(environment["CODEX_SQLITE_HOME"]) == codex_home / "state" + + +def test_reported_model_uses_claude_transcript_and_codex_command(tmp_path): + transcript = tmp_path / "events.jsonl" + transcript.write_text( + '{"type":"system","model":"claude-opus-exact"}\n' '{"type":"result","result":"done"}\n' + ) + + assert agent_qualification._reported_model(transcript, "claude") == ( + "claude-opus-exact", + "transcript", + ) + assert agent_qualification._reported_model(transcript, "codex") == ( + "gpt-5.6-sol", + "command", + ) diff --git a/tests/test_agent_qualification_generalization.py b/tests/test_agent_qualification_generalization.py new file mode 100644 index 0000000..8aed573 --- /dev/null +++ b/tests/test_agent_qualification_generalization.py @@ -0,0 +1,143 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from experiments import agent_qualification_generalization as generalization + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_plan_is_seeded_complete_and_excludes_v1_cases(): + manifest = { + "cases": [ + {"id": "c4-compatible", "status": "ready"}, + {"id": "c5-boundary", "status": "ready"}, + ] + } + + first = generalization.create_plan(manifest) + second = generalization.create_plan(manifest) + + assert first == second + assert first["entry_count"] == 24 + assert [entry["order"] for entry in first["entries"]] == list(range(1, 25)) + assert {entry["agent"] for entry in first["entries"]} == {"codex", "claude"} + assert {entry["condition"] for entry in first["entries"]} == {"docs", "skill"} + assert {entry["case_id"] for entry in first["entries"]} == { + "c4-compatible", + "c5-boundary", + } + assert all("c1" not in entry["id"] and "c3" not in entry["id"] for entry in first["entries"]) + + +def test_pending_cases_are_not_scheduled(): + manifest = { + "cases": [ + {"id": "ready", "status": "ready"}, + {"id": "pending", "status": "pending-gold"}, + ] + } + + plan = generalization.create_plan(manifest) + + assert plan["entry_count"] == 12 + assert {entry["case_id"] for entry in plan["entries"]} == {"ready"} + + +def test_docs_and_skill_prompts_are_text_identical_before_skill_injection(tmp_path, monkeypatch): + prompt = tmp_path / "prompt.txt" + prompt.write_text( + "case={{case_id}}\n" + "snapshot={{snapshot_id}}\n" + "revision={{revision}}\n" + "commands={{commands}}\n" + "claim={{declared_claim}}\n" + "excluded={{excluded_scope}}\n" + ) + monkeypatch.setattr(generalization, "PROMPT_PATH", prompt) + case = { + "id": "heldout", + "snapshot_id": "sha256:abc", + "revision": "pinned", + "commands": "native-test", + "declared_claim": "price-time", + "excluded_scope": "latency", + } + + docs = generalization.render_prompt(case, "docs") + skill = generalization.render_prompt(case, "skill") + + assert docs == skill + assert "tracebook-conformance==0.6.0" in docs + assert "{{" not in docs + + +def test_snapshot_digest_binds_file_mode_and_content(tmp_path): + source = tmp_path / "source" + source.mkdir() + file_path = source / "engine.txt" + file_path.write_text("price-time\n") + first = generalization._snapshot_digest(source) + + file_path.write_text("changed\n") + second = generalization._snapshot_digest(source) + file_path.write_text("price-time\n") + file_path.chmod(0o755) + third = generalization._snapshot_digest(source) + + assert len({first, second, third}) == 3 + + +def test_validate_tree_rejects_inventory_drift(tmp_path, monkeypatch): + private = tmp_path / "private" + source = private / "cache" + source.mkdir(parents=True) + (source / "dependency.bin").write_bytes(b"frozen") + manifests = private / "manifests" + manifests.mkdir() + snapshot_sha256 = "candidate-tree" + inventory = generalization._tree_inventory(source) + tree_sha256 = generalization._snapshot_digest(source) + tree_manifest = { + "kind": "frozen-tree-v1", + "case_id": "case", + "name": "dependency-cache", + "snapshot_sha256": snapshot_sha256, + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": sum(item["bytes"] for item in inventory), + "files": inventory, + } + manifest_path = manifests / "tree.json" + manifest_path.write_text(json.dumps(tree_manifest)) + monkeypatch.setattr(generalization, "REPOSITORY_ROOT", tmp_path) + monkeypatch.setattr(generalization, "PRIVATE_ROOT", private) + case = {"id": "case", "snapshot_sha256": snapshot_sha256} + declaration = { + "kind": "frozen-tree-v1", + "source": "private/cache", + "manifest": "private/manifests/tree.json", + "manifest_sha256": _sha256(manifest_path), + "tree_sha256": tree_sha256, + } + + generalization._validate_tree(case, declaration, name="dependency-cache") + (source / "dependency.bin").write_bytes(b"drift") + + with pytest.raises(generalization.GeneralizationError, match="inventory drifted"): + generalization._validate_tree(case, declaration, name="dependency-cache") + + +def test_freeze_refuses_to_overwrite_existing_bindings(tmp_path, monkeypatch): + manifest = tmp_path / "cases.json" + manifest.write_text("{}\n") + monkeypatch.setattr(generalization, "MANIFEST_PATH", manifest) + monkeypatch.setattr(generalization, "PLAN_PATH", tmp_path / "plan.json") + monkeypatch.setattr(generalization, "FREEZE_PATH", tmp_path / "freeze.json") + + with pytest.raises(generalization.GeneralizationError, match="refusing to overwrite"): + generalization.freeze() diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py new file mode 100644 index 0000000..d748a5e --- /dev/null +++ b/tests/test_agent_qualification_generalization_runs.py @@ -0,0 +1,250 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from experiments import agent_qualification_generalization_runs as execution + + +def test_plan_lookup_uses_exact_frozen_id(): + plan = {"entries": [{"id": "docs__case__codex__r1", "order": 1}]} + + assert execution._plan_entry(plan, "docs__case__codex__r1")["order"] == 1 + with pytest.raises(execution.ExecutionError, match="not one frozen plan entry"): + execution._plan_entry(plan, "missing") + + +def test_authorization_is_provider_and_case_specific(tmp_path, monkeypatch): + auth_root = tmp_path / "authorizations" + statement = "I approve C4 and C5 for Codex." + payload = { + "kind": execution.AUTHORIZATION_KIND, + "protocol": execution.PROTOCOL_ID, + "provider": "codex", + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + "user_statement": statement, + "statement_sha256": hashlib.sha256(statement.encode()).hexdigest(), + "recorded_at": "2026-08-01T00:00:00Z", + } + auth_root.mkdir() + (auth_root / "codex.json").write_text(json.dumps(payload)) + monkeypatch.setattr(execution, "AUTHORIZATION_ROOT", auth_root) + + assert execution.validate_authorization("codex")["provider"] == "codex" + with pytest.raises(execution.ExecutionError, match="cannot read claude authorization"): + execution.validate_authorization("claude") + + +def test_authorization_rejects_statement_drift(tmp_path, monkeypatch): + auth_root = tmp_path / "authorizations" + auth_root.mkdir() + payload = { + "kind": execution.AUTHORIZATION_KIND, + "protocol": execution.PROTOCOL_ID, + "provider": "codex", + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + "user_statement": "changed", + "statement_sha256": "not-the-hash", + "recorded_at": "2026-08-01T00:00:00Z", + } + (auth_root / "codex.json").write_text(json.dumps(payload)) + monkeypatch.setattr(execution, "AUTHORIZATION_ROOT", auth_root) + + with pytest.raises(execution.ExecutionError, match="statement binding drifted"): + execution.validate_authorization("codex") + + +def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch): + repository = tmp_path / "repo" + source = repository / "private" / "cache" + source.mkdir(parents=True) + (source / "artifact").write_text("frozen") + tree_hash = execution.helpers._snapshot_digest(source) + scratch = tmp_path / "scratch" + scratch.mkdir() + monkeypatch.setattr(execution, "REPOSITORY_ROOT", repository) + case = { + "id": "c5-boundary", + "dependency_cache": { + "source": "private/cache", + "tree_sha256": tree_hash, + "manifest_sha256": "manifest", + }, + } + + fixture = execution._copy_fixture(case, scratch) + + assert fixture["initial_tree_sha256"] == tree_hash + assert Path(fixture["target"]).name == "cargo-home" + assert execution.helpers._snapshot_digest(Path(fixture["target"])) == tree_hash + + +def test_native_surface_has_no_skill_in_docs_condition(tmp_path, monkeypatch): + root = tmp_path / "run" + workspace = root / "workspace" + scratch = root / "scratch" + workspace.mkdir(parents=True) + scratch.mkdir() + monkeypatch.setattr( + execution, + "_configure_delivery", + lambda: None, + ) + monkeypatch.setattr( + execution.delivery, + "_prepare_native_surface", + lambda **kwargs: ( + ["provider"], + { + "skill_installed": kwargs["condition"] == "skill", + "semantic_file_count": 1 if kwargs["condition"] == "skill" else 0, + }, + None, + ), + ) + + _, surface, _ = execution._native_surface( + provider="codex", + condition="docs", + workspace=workspace, + run_root=root, + scratch=scratch, + environment={}, + skill=b"skill", + skill_name="probe", + ) + + assert surface == {"skill_installed": False, "semantic_file_count": 0} + + +def test_validate_prior_runs_enforces_seeded_order(monkeypatch): + seen = [] + monkeypatch.setattr(execution, "validate_run", seen.append) + plan = { + "entries": [ + {"id": "first"}, + {"id": "second"}, + {"id": "third"}, + ] + } + + execution._validate_prior_runs(plan, 3) + + assert seen == ["first", "second"] + + +def test_execution_refuses_without_provider_authorization(monkeypatch): + monkeypatch.setattr(execution, "validate_freeze", lambda: {}) + monkeypatch.setattr( + execution, + "_load_primary", + lambda: ( + {"cases": []}, + { + "entries": [ + { + "id": "skill__c4-compatible__codex__r1", + "agent": "codex", + "case_id": "c4-compatible", + "condition": "skill", + "repetition": 1, + "order": 1, + } + ] + }, + {}, + ), + ) + monkeypatch.setattr( + execution, + "validate_authorization", + lambda provider: (_ for _ in ()).throw(execution.ExecutionError("authorization missing")), + ) + + with pytest.raises(execution.ExecutionError, match="authorization missing"): + execution.execute_run("skill__c4-compatible__codex__r1") + + +def test_wrapper_is_deterministic_and_refuses_drift(tmp_path, monkeypatch): + wrapper = tmp_path / ".claude-plugin" / "plugin.json" + monkeypatch.setattr(execution, "CLAUDE_WRAPPER_PATH", wrapper) + + execution._write_wrapper() + first = wrapper.read_bytes() + execution._write_wrapper() + wrapper.write_text("{}") + + assert json.loads(first) == execution._PLUGIN_MANIFEST + with pytest.raises(execution.ExecutionError, match="wrapper drifted"): + execution._write_wrapper() + + +def test_toolchain_environment_prepends_frozen_tools(tmp_path, monkeypatch): + maven = tmp_path / "maven" + java = tmp_path / "java" + python = tmp_path / "python" + uv = tmp_path / "uv" + for root in (maven, java, python): + (root / "bin").mkdir(parents=True) + uv.write_text("binary") + monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven)) + monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java)) + monkeypatch.setenv("TRACEBOOK_V2_PYTHON_HOME", str(python)) + monkeypatch.setenv("TRACEBOOK_V2_UV_BINARY", str(uv)) + monkeypatch.setenv("PATH", "/usr/bin:/bin") + + environment = execution._toolchain_environment() + + assert environment["JAVA_HOME"] == str(java) + assert environment["PATH"].split(":")[:4] == [ + str(maven / "bin"), + str(java / "bin"), + str(python / "bin"), + str(tmp_path), + ] + + +def test_provider_shells_receive_fresh_paths(tmp_path, monkeypatch): + monkeypatch.delenv("TRACEBOOK_V2_MAVEN_HOME", raising=False) + monkeypatch.delenv("TRACEBOOK_V2_JAVA_HOME", raising=False) + monkeypatch.delenv("TRACEBOOK_V2_PYTHON_HOME", raising=False) + monkeypatch.delenv("TRACEBOOK_V2_UV_BINARY", raising=False) + + values = execution._codex_shell_environment(tmp_path) + + assert values["FRESH_M2"] == str(tmp_path / "m2-repository") + assert values["FRESH_CARGO_HOME"] == str(tmp_path / "cargo-home") + assert values["FRESH_TARGET"] == str(tmp_path / "cargo-target") + + +def test_shakedown_requires_first_party_claude_route(monkeypatch): + def binding(provider): + if provider == "claude": + return { + "auth_route": { + "loggedIn": True, + "authMethod": "third_party", + "apiProvider": "bedrock", + } + } + return {"requested_model": execution.EXPECTED_MODELS[provider]} + + monkeypatch.setattr(execution, "_provider_binding", binding) + + with pytest.raises(execution.ExecutionError, match="first-party route"): + execution._validated_provider_bindings() diff --git a/tests/test_agent_qualification_treatments.py b/tests/test_agent_qualification_treatments.py new file mode 100644 index 0000000..dbd31cd --- /dev/null +++ b/tests/test_agent_qualification_treatments.py @@ -0,0 +1,311 @@ +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from experiments import agent_qualification +from experiments import agent_qualification_treatments as treatments + +BASELINE_RUNNER_SHA256 = "6ccf1f0ceee34672e049c428cb368d98a1c36270aeff21900f25b79e659261ef" + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_frozen_baseline_runner_is_byte_identical(): + assert _sha256(agent_qualification.RUNNER_PATH) == BASELINE_RUNNER_SHA256 + + +def test_treatment_prompt_is_identical_between_docs_and_skill(): + manifest = agent_qualification.validate_manifest(agent_qualification.DEFAULT_MANIFEST_PATH) + for case in manifest["cases"]: + if case["status"] != "ready": + continue + rendered = treatments.render_prompt(case) + assert rendered == agent_qualification.render_prompt(case, "docs") + assert rendered == agent_qualification.render_prompt(case, "skill") + assert "tracebook-conformance==0.6.0" in rendered + + +def test_master_plan_is_two_matched_blocks_in_baseline_order(): + manifest = agent_qualification.validate_manifest(agent_qualification.DEFAULT_MANIFEST_PATH) + entries = treatments._master_entries(manifest) + baseline_plan = json.loads(agent_qualification.DEFAULT_PLAN_PATH.read_text()) + baseline_cells = [ + { + "case_id": entry["case_id"], + "agent": entry["agent"], + "repetition": entry["repetition"], + } + for entry in baseline_plan["entries"] + ] + + assert len(entries) == 24 + assert [entry["condition"] for entry in entries[:12]] == ["docs"] * 12 + assert [entry["condition"] for entry in entries[12:]] == ["skill"] * 12 + for offset in (0, 12): + assert [ + { + "case_id": entry["case_id"], + "agent": entry["agent"], + "repetition": entry["repetition"], + } + for entry in entries[offset : offset + 12] + ] == baseline_cells + assert len({entry["run_id"] for entry in entries}) == 24 + assert len({entry["evidence_id"] for entry in entries}) == 24 + + +def test_skill_is_candidate_agnostic_and_exactly_one_measured_file(): + manifest = agent_qualification.validate_manifest(agent_qualification.DEFAULT_MANIFEST_PATH) + skill = treatments._load_skill(treatments.DEFAULT_SKILL_PATH, manifest) + + assert skill["name"] == treatments.SKILL_NAME + assert skill["sha256"] == _sha256(treatments.DEFAULT_SKILL_PATH) + assert b"heldout" not in skill["content"].lower() + assert b"order-matcher" not in skill["content"].lower() + assert b"matching-engine-rs" not in skill["content"].lower() + + +def test_skill_leakage_audit_rejects_heldout_identifier(tmp_path, monkeypatch): + monkeypatch.setattr(treatments, "PRIVATE_ROOT", tmp_path) + skill = tmp_path / "SKILL.md" + skill.write_text( + "---\n" + f"name: {treatments.SKILL_NAME}\n" + "description: synthetic\n" + "---\n" + "Target c1-heldout directly.\n" + ) + manifest = {"cases": [{"id": "c1-heldout"}]} + + with pytest.raises(treatments.EvaluationError, match="leakage audit"): + treatments._load_skill(skill, manifest) + + +def test_private_output_rejects_escape_before_creating_parent(tmp_path, monkeypatch): + private_root = tmp_path / "private" + private_root.mkdir() + outside = tmp_path / "outside" / "result.json" + monkeypatch.setattr(treatments, "PRIVATE_ROOT", private_root) + + with pytest.raises(treatments.EvaluationError, match="must stay inside"): + treatments._private_output(outside, "test output") + + assert not outside.parent.exists() + + +def test_master_plan_shape_rejects_tampering(): + entries = [ + { + "ordinal": index, + "run_id": f"run-{index}", + "evidence_id": f"evidence-{index}", + "condition": "docs" if index <= 12 else "skill", + } + for index in range(1, 25) + ] + assert len(treatments._validate_plan_entry_shape(entries)) == 24 + entries[12]["evidence_id"] = entries[0]["evidence_id"] + + with pytest.raises(treatments.EvaluationError, match="evidence IDs"): + treatments._validate_plan_entry_shape(entries) + + +def test_claude_treatment_command_is_skill_capable_and_customization_isolated( + tmp_path, +): + command = treatments._claude_treatment_command(tmp_path / "settings.json", tmp_path / "plugin") + + assert "--safe-mode" not in command + assert "--disable-slash-commands" not in command + assert command[command.index("--setting-sources") + 1] == "" + assert command[command.index("--plugin-dir") + 1] == str(tmp_path / "plugin") + assert "Skill" in command[command.index("--tools") + 1].split(",") + assert command[command.index("--model") + 1] == "claude-opus-4-8" + assert command[command.index("--mcp-config") + 1] == '{"mcpServers":{}}' + + +def test_claude_wrapper_tree_contains_no_semantic_payload(): + root = treatments.DEFAULT_CLAUDE_WRAPPER_PATH.parents[1] + record = treatments._tree_record(root) + + assert record["file_count"] == 1 + assert record["files"][0]["path"] == ".claude-plugin/plugin.json" + + +def test_claude_settings_make_plugin_read_only_to_agent(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + plugin = scratch / "plugin" + workspace.mkdir() + scratch.mkdir() + plugin.mkdir() + + settings = treatments._claude_treatment_settings(workspace, scratch, plugin) + + assert "CLAUDE_CONFIG_DIR" not in settings["env"] + assert "HOME" not in settings["env"] + assert settings["env"]["ZDOTDIR"] == str(scratch / "zsh-env") + assert (scratch / "zsh-env" / ".zshenv").read_text() == ( + f"export HOME={scratch / 'shell-home'}\n" + ) + assert str(plugin) in settings["sandbox"]["filesystem"]["denyWrite"] + assert f"Edit(/{plugin}/**)" in settings["permissions"]["deny"] + + +def test_codex_native_surface_differs_only_by_skill_file(tmp_path, monkeypatch): + auth = tmp_path / "auth.json" + auth.write_text("{}\n") + monkeypatch.setattr(agent_qualification, "CODEX_AUTH_PATH", auth) + workspace = tmp_path / "workspace" + workspace.mkdir() + skill = treatments.DEFAULT_SKILL_PATH.read_bytes() + + docs_root = tmp_path / "docs" + docs_scratch = docs_root / "scratch" + docs_root.mkdir() + docs_scratch.mkdir() + docs_environment: dict[str, str] = {} + docs_command, docs_surface, _ = treatments._prepare_native_surface( + agent="codex", + condition="docs", + workspace=workspace, + external_run_root=docs_root, + scratch=docs_scratch, + environment=docs_environment, + skill_content=skill, + ) + + skill_root = tmp_path / "skill" + skill_scratch = skill_root / "scratch" + skill_root.mkdir() + skill_scratch.mkdir() + skill_environment: dict[str, str] = {} + skill_command, skill_surface, _ = treatments._prepare_native_surface( + agent="codex", + condition="skill", + workspace=workspace, + external_run_root=skill_root, + scratch=skill_scratch, + environment=skill_environment, + skill_content=skill, + ) + + assert docs_surface["skill_installed"] is False + assert skill_surface["skill_installed"] is True + assert not (docs_root / "codex-home" / "skills" / treatments.SKILL_NAME).exists() + installed = skill_root / "codex-home" / "skills" / treatments.SKILL_NAME / "SKILL.md" + assert installed.read_bytes() == skill + for command, root in ((docs_command, docs_root), (skill_command, skill_root)): + permission = next(item for item in command if item.startswith("permissions.agent-eval=")) + assert f'"{root / "codex-home" / "skills"}"="read"' in permission + + +def test_native_surface_final_audit_detects_absence_and_drift(tmp_path): + skill = treatments.DEFAULT_SKILL_PATH.read_bytes() + external_root = tmp_path / "run" + scratch = external_root / "scratch" + scratch.mkdir(parents=True) + + docs = treatments._audit_native_surface( + agent="claude", + condition="docs", + external_run_root=external_root, + scratch=scratch, + skill_content=skill, + ) + assert docs["mutated"] is False + + path = scratch / "claude-plugin" / "skills" / treatments.SKILL_NAME / "SKILL.md" + path.parent.mkdir(parents=True) + path.write_bytes(b"drift") + treatment = treatments._audit_native_surface( + agent="claude", + condition="skill", + external_run_root=external_root, + scratch=scratch, + skill_content=skill, + ) + assert treatment["mutated"] is True + + +def test_transcript_terminal_state_requires_clean_completion(tmp_path): + codex = tmp_path / "codex.jsonl" + codex.write_text('{"type":"turn.completed"}\n') + claude = tmp_path / "claude.jsonl" + claude.write_text( + '{"type":"result","terminal_reason":"completed","subtype":"success",' '"result":"done"}\n' + ) + failed = tmp_path / "failed.jsonl" + failed.write_text('{"type":"error","message":"network"}\n') + + assert treatments._transcript_terminal_state(codex, "codex") == "completed" + assert treatments._transcript_terminal_state(claude, "claude") == "completed" + assert treatments._transcript_terminal_state(failed, "codex") == "failed" + + +def test_claude_catalog_audit_requires_only_wrapper_and_expected_skill(tmp_path): + transcript = tmp_path / "claude.jsonl" + transcript.write_text( + json.dumps( + { + "type": "system", + "subtype": "init", + "model": "claude-opus-4-8", + "plugins": [{"name": "agent-qualification-treatment"}], + "skills": [f"agent-qualification-treatment:{treatments.SKILL_NAME}"], + "mcp_servers": [], + "tools": [ + "Bash", + "Edit", + "Glob", + "Grep", + "Read", + "Skill", + "WebFetch", + "WebSearch", + "Write", + ], + } + ) + + "\n" + ) + + assert treatments._claude_catalog_audit(transcript, condition="skill")["valid"] is True + assert treatments._claude_catalog_audit(transcript, condition="docs")["valid"] is False + + +def test_measurement_policy_uses_conservative_paired_timing_gate(): + timing = treatments.MEASUREMENT_POLICY["timing_gate"] + + assert timing["minimum_matched_autonomous_safe_pass_pairs"] == 3 + assert timing["both_cases_required"] is True + assert timing["skill_safe_pass_rate_must_not_be_lower"] is True + assert timing["pass_threshold"] == 0.70 + + +def test_official_run_rejects_timeout_drift_before_freeze_validation(): + with pytest.raises(treatments.EvaluationError, match="timeout must remain"): + treatments.execute_run("not-a-run", timeout_seconds=1) + + +def test_direct_script_entrypoint_smoke(): + completed = subprocess.run( + [ + sys.executable, + str(treatments.RUNNER_PATH), + "--help", + ], + cwd=treatments.REPOSITORY_ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0 + assert "create the immutable 24-run plan" in completed.stdout From 7fa31bb97472c2c3a0020faade01e0cbaf6d1833 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Sat, 1 Aug 2026 21:42:01 -0400 Subject: [PATCH 02/19] make experiment tests hermetic in CI --- tests/test_agent_qualification_experiment.py | 45 ++++++++------ tests/test_agent_qualification_treatments.py | 62 +++++++++++++++----- 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/tests/test_agent_qualification_experiment.py b/tests/test_agent_qualification_experiment.py index 97b077a..bf3564c 100644 --- a/tests/test_agent_qualification_experiment.py +++ b/tests/test_agent_qualification_experiment.py @@ -227,13 +227,21 @@ def test_claude_permissions_delegate_paths_to_fail_closed_sandbox(tmp_path): assert str(workspace) in settings["sandbox"]["filesystem"]["allowWrite"] assert str(workspace / ".git") in settings["sandbox"]["filesystem"]["denyWrite"] assert str(Path.home()) in settings["sandbox"]["filesystem"]["denyRead"] - assert str(Path("/usr/local/Cellar")) in settings["sandbox"]["filesystem"]["allowRead"] - assert settings["env"]["PATH"].startswith( - str(Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent) - ) - assert settings["env"]["DOTNET_ROOT"] == str( - Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent.parent / "libexec" - ) + for candidate in ( + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + ): + assert (str(candidate) in settings["sandbox"]["filesystem"]["allowRead"]) is ( + candidate.exists() + ) + _, dotnet_launcher_dir, dotnet_root = agent_qualification._dotnet_layout() + if dotnet_launcher_dir is None: + assert "DOTNET_ROOT" not in settings["env"] + else: + assert settings["env"]["PATH"].startswith(str(dotnet_launcher_dir)) + assert settings["env"]["DOTNET_ROOT"] == str(dotnet_root) assert settings["env"]["NuGetAudit"] == "false" @@ -264,10 +272,13 @@ def test_codex_command_uses_pure_permission_profile(tmp_path): assert '":root"="deny"' in permission_override assert '":workspace_roots"={"."="write",".git"="read"}' in permission_override assert f'{scratch}"="write"' in permission_override - assert '"/usr/local/bin/dotnet"="read"' in permission_override - assert '"/usr/local/share/dotnet"="read"' in permission_override - assert '"/usr/local/opt"="read"' in permission_override - assert '"/usr/local/Cellar"="read"' in permission_override + for candidate in ( + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + ): + assert (f'"{candidate}"="read"' in permission_override) is candidate.exists() assert 'mode="limited"' in permission_override assert "allow_upstream_proxy=false" in permission_override assert "allow_local_binding=true" not in permission_override @@ -277,12 +288,12 @@ def test_codex_command_uses_pure_permission_profile(tmp_path): assert any(item.startswith("shell_environment_policy.set=") for item in command) assert "--search" in command shell_environment = agent_qualification._codex_shell_environment(scratch) - assert shell_environment["PATH"].startswith( - str(Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent) - ) - assert shell_environment["DOTNET_ROOT"] == str( - Path("/usr/local/opt/dotnet@8/bin/dotnet").resolve().parent.parent / "libexec" - ) + _, dotnet_launcher_dir, dotnet_root = agent_qualification._dotnet_layout() + if dotnet_launcher_dir is None: + assert "DOTNET_ROOT" not in shell_environment + else: + assert shell_environment["PATH"].startswith(str(dotnet_launcher_dir)) + assert shell_environment["DOTNET_ROOT"] == str(dotnet_root) def test_clean_environment_removes_secret_shaped_names(tmp_path, monkeypatch): diff --git a/tests/test_agent_qualification_treatments.py b/tests/test_agent_qualification_treatments.py index dbd31cd..78ca1f6 100644 --- a/tests/test_agent_qualification_treatments.py +++ b/tests/test_agent_qualification_treatments.py @@ -1,5 +1,6 @@ import hashlib import json +import random import subprocess import sys from pathlib import Path @@ -16,12 +17,29 @@ def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def _synthetic_manifest(): + return { + "cases": [ + { + "id": case_id, + "status": "ready", + "snapshot_id": f"sha256:{case_id}", + "revision": f"revision-{case_id}", + "commands": "native-test", + "declared_claim": "price-time matching", + "excluded_scope": "production readiness", + } + for case_id in ("case-alpha", "case-beta") + ] + } + + def test_frozen_baseline_runner_is_byte_identical(): assert _sha256(agent_qualification.RUNNER_PATH) == BASELINE_RUNNER_SHA256 def test_treatment_prompt_is_identical_between_docs_and_skill(): - manifest = agent_qualification.validate_manifest(agent_qualification.DEFAULT_MANIFEST_PATH) + manifest = _synthetic_manifest() for case in manifest["cases"]: if case["status"] != "ready": continue @@ -32,17 +50,19 @@ def test_treatment_prompt_is_identical_between_docs_and_skill(): def test_master_plan_is_two_matched_blocks_in_baseline_order(): - manifest = agent_qualification.validate_manifest(agent_qualification.DEFAULT_MANIFEST_PATH) + manifest = _synthetic_manifest() entries = treatments._master_entries(manifest) - baseline_plan = json.loads(agent_qualification.DEFAULT_PLAN_PATH.read_text()) baseline_cells = [ { - "case_id": entry["case_id"], - "agent": entry["agent"], - "repetition": entry["repetition"], + "case_id": case["id"], + "agent": agent, + "repetition": repetition, } - for entry in baseline_plan["entries"] + for case in manifest["cases"] + for agent in ("codex", "claude") + for repetition in range(1, treatments.REPETITIONS + 1) ] + random.Random(agent_qualification.RANDOMIZATION_SEED).shuffle(baseline_cells) assert len(entries) == 24 assert [entry["condition"] for entry in entries[:12]] == ["docs"] * 12 @@ -60,12 +80,21 @@ def test_master_plan_is_two_matched_blocks_in_baseline_order(): assert len({entry["evidence_id"] for entry in entries}) == 24 -def test_skill_is_candidate_agnostic_and_exactly_one_measured_file(): - manifest = agent_qualification.validate_manifest(agent_qualification.DEFAULT_MANIFEST_PATH) - skill = treatments._load_skill(treatments.DEFAULT_SKILL_PATH, manifest) +def test_skill_is_candidate_agnostic_and_exactly_one_measured_file(tmp_path, monkeypatch): + monkeypatch.setattr(treatments, "PRIVATE_ROOT", tmp_path) + skill_path = tmp_path / "skill" / "SKILL.md" + skill_path.parent.mkdir() + skill_path.write_text( + "---\n" + f"name: {treatments.SKILL_NAME}\n" + "description: Candidate-agnostic qualification workflow.\n" + "---\n\n" + "Inspect the supplied candidate and preserve identity lifecycles.\n" + ) + skill = treatments._load_skill(skill_path, _synthetic_manifest()) assert skill["name"] == treatments.SKILL_NAME - assert skill["sha256"] == _sha256(treatments.DEFAULT_SKILL_PATH) + assert skill["sha256"] == _sha256(skill_path) assert b"heldout" not in skill["content"].lower() assert b"order-matcher" not in skill["content"].lower() assert b"matching-engine-rs" not in skill["content"].lower() @@ -130,8 +159,11 @@ def test_claude_treatment_command_is_skill_capable_and_customization_isolated( assert command[command.index("--mcp-config") + 1] == '{"mcpServers":{}}' -def test_claude_wrapper_tree_contains_no_semantic_payload(): - root = treatments.DEFAULT_CLAUDE_WRAPPER_PATH.parents[1] +def test_claude_wrapper_tree_contains_no_semantic_payload(tmp_path): + root = tmp_path / "claude-wrapper" + manifest = root / ".claude-plugin" / "plugin.json" + manifest.parent.mkdir(parents=True) + manifest.write_text('{"name":"agent-qualification-treatment","version":"1.0.0"}\n') record = treatments._tree_record(root) assert record["file_count"] == 1 @@ -164,7 +196,7 @@ def test_codex_native_surface_differs_only_by_skill_file(tmp_path, monkeypatch): monkeypatch.setattr(agent_qualification, "CODEX_AUTH_PATH", auth) workspace = tmp_path / "workspace" workspace.mkdir() - skill = treatments.DEFAULT_SKILL_PATH.read_bytes() + skill = b"synthetic frozen skill\n" docs_root = tmp_path / "docs" docs_scratch = docs_root / "scratch" @@ -207,7 +239,7 @@ def test_codex_native_surface_differs_only_by_skill_file(tmp_path, monkeypatch): def test_native_surface_final_audit_detects_absence_and_drift(tmp_path): - skill = treatments.DEFAULT_SKILL_PATH.read_bytes() + skill = b"synthetic frozen skill\n" external_root = tmp_path / "run" scratch = external_root / "scratch" scratch.mkdir(parents=True) From 50ac3b63c934ec9f3c18869cc5c68b221b73fdd2 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Sat, 1 Aug 2026 22:09:05 -0400 Subject: [PATCH 03/19] force first-party Claude experiment routing --- ...alification-generalization-execution-v2.md | 5 ++ ...agent_qualification_generalization_runs.py | 21 +++++++- ...agent_qualification_generalization_runs.py | 48 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md index 3454b4c..238a7ca 100644 --- a/docs/agent-qualification-generalization-execution-v2.md +++ b/docs/agent-qualification-generalization-execution-v2.md @@ -20,6 +20,11 @@ Before the execution layer may be frozen, the harness must: 4. leave provider authorization outside the immutable freeze so authorization can be recorded later without changing experimental inputs. +The Claude binding probe and every Claude run explicitly disable the +Bedrock, Vertex, and Foundry routing flags in their isolated settings. The +harness still refuses to freeze unless `claude auth status` identifies an +authenticated `firstParty` route. + The synthetic canary contains no candidate source, Tracebook guidance, gold data, or measured prompt material. diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py index 48da387..ade1d11 100644 --- a/experiments/agent_qualification_generalization_runs.py +++ b/experiments/agent_qualification_generalization_runs.py @@ -60,6 +60,11 @@ DEFAULT_TIMEOUT_SECONDS = 120 * 60 EXPECTED_MODELS = {"codex": "gpt-5.6-sol", "claude": "claude-opus-4-8"} PROVIDERS = tuple(EXPECTED_MODELS) +CLAUDE_FIRST_PARTY_ENV = { + "CLAUDE_CODE_USE_BEDROCK": "0", + "CLAUDE_CODE_USE_VERTEX": "0", + "CLAUDE_CODE_USE_FOUNDRY": "0", +} _PLUGIN_MANIFEST = { "author": {"name": "Tracebook evaluator"}, @@ -165,8 +170,21 @@ def _provider_binding(provider: str) -> dict[str, Any]: "requested_model": EXPECTED_MODELS[provider], } if provider == "claude": + route_settings = json.dumps( + {"env": CLAUDE_FIRST_PARTY_ENV}, + sort_keys=True, + separators=(",", ":"), + ) completed = subprocess.run( - (str(binary), "auth", "status"), + ( + str(binary), + "--setting-sources", + "", + "--settings", + route_settings, + "auth", + "status", + ), check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -254,6 +272,7 @@ def _codex_shell_environment(scratch: Path) -> dict[str, str]: def _claude_settings(workspace: Path, scratch: Path, plugin_root: Path) -> dict[str, Any]: settings = delivery._claude_treatment_settings(workspace, scratch, plugin_root) + settings["env"].update(CLAUDE_FIRST_PARTY_ENV) settings["env"].update(_toolchain_environment()) settings["env"].update(_fresh_environment(scratch)) return settings diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py index d748a5e..fc9883a 100644 --- a/tests/test_agent_qualification_generalization_runs.py +++ b/tests/test_agent_qualification_generalization_runs.py @@ -232,6 +232,54 @@ def test_provider_shells_receive_fresh_paths(tmp_path, monkeypatch): assert values["FRESH_TARGET"] == str(tmp_path / "cargo-target") +def test_claude_binding_probes_an_explicit_first_party_route(tmp_path, monkeypatch): + binary = tmp_path / "claude" + binary.write_text("claude") + observed = {} + + monkeypatch.setattr(execution, "_provider_binary", lambda provider: binary) + monkeypatch.setattr(execution.helpers, "_binary_version", lambda *args: "2.1.220") + + def run(command, **kwargs): + observed["command"] = command + return execution.subprocess.CompletedProcess( + command, + 0, + stdout=json.dumps( + { + "loggedIn": True, + "authMethod": "claude.ai", + "apiProvider": "firstParty", + "subscriptionType": "max", + } + ), + ) + + monkeypatch.setattr(execution.subprocess, "run", run) + + binding = execution._provider_binding("claude") + + command = observed["command"] + assert command[1:4] == ("--setting-sources", "", "--settings") + assert json.loads(command[4])["env"] == execution.CLAUDE_FIRST_PARTY_ENV + assert command[5:] == ("auth", "status") + assert binding["auth_route"]["apiProvider"] == "firstParty" + + +def test_claude_run_settings_disable_third_party_routes(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + plugin_root = tmp_path / "plugin" + workspace.mkdir() + scratch.mkdir() + plugin_root.mkdir() + + settings = execution._claude_settings(workspace, scratch, plugin_root) + + for name, value in execution.CLAUDE_FIRST_PARTY_ENV.items(): + assert settings["env"][name] == value + + def test_shakedown_requires_first_party_claude_route(monkeypatch): def binding(provider): if provider == "claude": From 361d95d99c85da16a2a214f5dd8b3d0a7040901e Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Mon, 3 Aug 2026 11:52:33 -0400 Subject: [PATCH 04/19] publish agent qualification v2 results --- ...qualification-generalization-v2-results.md | 96 +++++++++++++++++++ docs/agent-qualification-generalization-v2.md | 10 +- 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 docs/agent-qualification-generalization-v2-results.md diff --git a/docs/agent-qualification-generalization-v2-results.md b/docs/agent-qualification-generalization-v2-results.md new file mode 100644 index 0000000..5904257 --- /dev/null +++ b/docs/agent-qualification-generalization-v2-results.md @@ -0,0 +1,96 @@ +# Agentic Qualification Generalization V2 Results + +**Protocol:** `agent-qualification-generalization-v2` +**Public protocol commit:** `50ac3b63c934ec9f3c18869cc5c68b221b73fdd2` +**Execution completed:** 2026-08-02 +**Aggregate grading completed:** 2026-08-03 + +## Primary Result + +All 24 preregistered runs received valid pre-score technical verdicts before +semantic outcomes were inspected. The frozen skill improved the autonomous +safe-pass rate by 50 percentage points over public documentation alone. + +| Condition | Safe passes | Runs | Rate | +| --- | ---: | ---: | ---: | +| Public docs | 1 | 12 | 8.3% | +| Frozen skill | 7 | 12 | 58.3% | + +The result exceeds the preregistered directional-product-signal threshold of ++20 percentage points. The skill condition produced zero unsupported +candidate-defect accusations, and no measured run changed tracked production +matching behavior. + +## Required Breakdowns + +### By agent family + +| Agent | Docs | Skill | Delta | +| --- | ---: | ---: | ---: | +| Claude | 1/6 (16.7%) | 3/6 (50.0%) | +33.3 pp | +| Codex | 0/6 (0.0%) | 4/6 (66.7%) | +66.7 pp | + +### By held-out case class + +| Case | Docs | Skill | Delta | +| --- | ---: | ---: | ---: | +| Compatible engine | 0/6 (0.0%) | 1/6 (16.7%) | +16.7 pp | +| Profile boundary | 1/6 (16.7%) | 6/6 (100.0%) | +83.3 pp | + +The intervention generalized strongly to boundary recognition: every skill +run correctly refused to manufacture Tracebook qualification evidence for the +system that did not select identified resting venue makers. Compatible-engine +release evidence remained the bottleneck. Only one of six skill runs satisfied +every clean-build, candidate-identity, canonical-bundle, and byte-equality +gate. + +## Interpretation + +This is a positive directional product result, not a statistically conclusive +model comparison. It supports three narrow claims: + +1. an explicit qualification skill materially improved safe autonomous + workflow execution in this held-out cohort; +2. the improvement was driven mainly by profile-boundary judgment; and +3. compatible-engine release evidence still required machine-enforced gates + rather than more prose. + +It does not establish broad model superiority, production adoption, product- +market fit, or reliable autonomous qualification across matching engines. The +adaptive v2 cohort must not be pooled with v1 as one untouched confirmatory +study, revised after grading, or rerun to improve the result. + +## Product Consequence + +The highest-leverage follow-up is a public evidence workflow that: + +- pins candidate name, revision, and source snapshot before adapter readiness; +- creates two separate candidate, adapter, build, cache, and output roots; +- requires two canonical qualification bundles with matching deterministic + identifiers, counts, coverage, and bytes; and +- emits one concise, grading-ready manifest. + +That follow-up is implemented separately in +[PR #71](https://github.com/Taz33m/tracebook/pull/71). It is not part of the +frozen intervention and must be evaluated on another untouched candidate. + +## Audit Trail + +Evaluator-only case identities, gold, authorizations, provider transcripts, +workspaces, and per-run grading remain under the ignored private experiment +tree. They are intentionally not published with the protocol. The aggregate +values above were transcribed from the completed private scorecard whose +SHA-256 is: + +```text +7741a127949b0bf8881b8827e1ec46ee12082a40c84f7bc8ba6b25ba409269b7 +``` + +The private narrative report used for the transcription has SHA-256: + +```text +cfbf53af662407bbe8df74970cbaf107573d74d3d3ce8881976f3103ca2de78d +``` + +These hashes bind the public aggregate report to the preserved evaluator +record without publishing restricted experiment materials. diff --git a/docs/agent-qualification-generalization-v2.md b/docs/agent-qualification-generalization-v2.md index b782257..d7dd850 100644 --- a/docs/agent-qualification-generalization-v2.md +++ b/docs/agent-qualification-generalization-v2.md @@ -1,9 +1,13 @@ # Agentic Qualification Generalization Evaluation **Protocol:** `agent-qualification-generalization-v2` -**Freeze date:** pending executable freeze -**Status:** design frozen only after `freeze.json` validates; provider execution -requires case-specific authorization +**Historical status:** frozen before measured execution; all 24 preregistered +runs received valid pre-score verdicts on 2026-08-02 and aggregate semantic +grading completed on 2026-08-03 + +This document preserves the preregistered design. The aggregate outcome and +claim boundaries are reported separately in the +[v2 results note](agent-qualification-generalization-v2-results.md). ## Question From fdda996130cf4962eda7ce3bbe9f607f1847f579 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Tue, 25 Aug 2026 01:20:36 -0400 Subject: [PATCH 05/19] fix agent qualification experiment guards --- ...alification-generalization-execution-v2.md | 17 ++ ...qualification-generalization-v2-results.md | 9 +- docs/agent-qualification-generalization-v2.md | 22 +- experiments/agent_qualification.py | 101 ++++++++- .../agent_qualification_generalization.py | 39 ++-- ...agent_qualification_generalization_runs.py | 210 +++++++++++++++--- experiments/agent_qualification_treatments.py | 73 ++++-- tests/test_agent_qualification_experiment.py | 60 +++++ ...test_agent_qualification_generalization.py | 18 ++ ...agent_qualification_generalization_runs.py | 131 ++++++++++- tests/test_agent_qualification_treatments.py | 66 +++++- 11 files changed, 661 insertions(+), 85 deletions(-) diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md index 238a7ca..720677e 100644 --- a/docs/agent-qualification-generalization-execution-v2.md +++ b/docs/agent-qualification-generalization-execution-v2.md @@ -25,6 +25,10 @@ Bedrock, Vertex, and Foundry routing flags in their isolated settings. The harness still refuses to freeze unless `claude auth status` identifies an authenticated `firstParty` route. +Every provider run also starts without user or project configuration. The +harness supplies a fixed minimal system `PATH`, adds only the toolchain prefixes +bound by the execution freeze, and grants read access to those exact roots. + The synthetic canary contains no candidate source, Tracebook guidance, gold data, or measured prompt material. @@ -43,6 +47,10 @@ seed, new build target, isolated provider state, and condition-specific native skill surface. The local Tracebook checkout, private gold, prior workspaces, and prior transcripts are not mounted into the workspace. +Before accepting a Claude turn, the harness audits the provider catalog and +requires no MCP servers, exactly one intended plugin, and skill visibility that +matches the assigned condition. The audit is preserved with the run. + The harness executes only the next entry in the frozen 24-run order. A later entry is rejected until every earlier entry has a valid pre-score technical verdict. Existing run or external-workspace paths are never overwritten. @@ -55,11 +63,20 @@ reported model before semantic grading. A pre-score verdict covers technical validity only; it must not inspect whether the agent reached the correct qualification outcome. +Fixture validity requires the final dependency-cache hash to equal its frozen +initial hash. Workspace inventory and patch capture include ignored candidate +files, using a temporary Git index so the evaluator's real index is unchanged. + A completed provider turn, refusal, timeout, or semantic failure is not restartable. Only a genuine technical interruption that lacks a completed provider turn may be archived intact and restarted. Archives remain under the ignored private experiment tree. +If execution fails before a technical verdict exists, the harness removes the +new external workspace and quarantines the incomplete run root under the +private interruption archive. A documented clean restart is therefore possible +without overwriting evidence. + ## Commands The public executor is diff --git a/docs/agent-qualification-generalization-v2-results.md b/docs/agent-qualification-generalization-v2-results.md index 5904257..4287606 100644 --- a/docs/agent-qualification-generalization-v2-results.md +++ b/docs/agent-qualification-generalization-v2-results.md @@ -4,8 +4,15 @@ **Public protocol commit:** `50ac3b63c934ec9f3c18869cc5c68b221b73fdd2` **Execution completed:** 2026-08-02 **Aggregate grading completed:** 2026-08-03 +**Validity:** invalidated on 2026-08-25 after execution-harness review; retained +only as a historical audit record pending a corrected 24-run recollection -## Primary Result +## Historical Result (Invalidated) + +The figures below describe the original execution. They must not be used for +inference: the corrected harness restores fixture-mutation validation, Claude +provider-catalog auditing, and the restricted toolchain environment, among +other isolation and evidence-capture fixes. All 24 preregistered runs received valid pre-score technical verdicts before semantic outcomes were inspected. The frozen skill improved the autonomous diff --git a/docs/agent-qualification-generalization-v2.md b/docs/agent-qualification-generalization-v2.md index d7dd850..c33a694 100644 --- a/docs/agent-qualification-generalization-v2.md +++ b/docs/agent-qualification-generalization-v2.md @@ -1,9 +1,9 @@ # Agentic Qualification Generalization Evaluation **Protocol:** `agent-qualification-generalization-v2` -**Historical status:** frozen before measured execution; all 24 preregistered -runs received valid pre-score verdicts on 2026-08-02 and aggregate semantic -grading completed on 2026-08-03 +**Correction status:** the 2026-08-02 execution is retained as historical +evidence but is not valid for inference. A corrected harness must collect a +fresh 24-run matrix before the aggregate claim is reinstated. This document preserves the preregistered design. The aggregate outcome and claim boundaries are reported separately in the @@ -64,7 +64,8 @@ Each frozen case binds: - owner claim and excluded scope; - supported native commands; - evaluator-only gold; -- a frozen writable dependency-cache seed; and +- a frozen writable dependency-cache seed and manifest-declared destination; + and - applicable evaluator adapter and machine evidence. Agents may inspect only their fresh candidate copy, the frozen prompt, public @@ -118,6 +119,10 @@ Before timing, the harness must: 5. initialize a fresh Git repository only for diff capture; and 6. record the initial source, dependency, and workspace tree hashes. +After the provider turn, the harness rehashes the dependency seed and rejects +the run if it differs from the initial hash. The destination is read from the +frozen case manifest rather than inferred from a case identifier. + The agent must produce two independent evidence runs. A valid pair requires: - a rebuilt adapter or native target for each run; @@ -181,6 +186,13 @@ Pre-registered interpretation: - Gold, frozen adapters, prior qualification bundles, v1 runs, OpenWiki, and the local Tracebook checkout are not mounted into agent workspaces. - Candidate build scripts run only inside the agent sandbox. +- The provider processes receive a fixed minimal system `PATH`; only frozen + native-toolchain prefixes may be added, and those exact roots are included + in each provider's filesystem-read policy. +- Provider user and project settings are disabled. Claude's visible provider + catalog is audited before a run can receive a valid pre-score verdict: no MCP + servers, exactly the intended plugin surface, and the expected skill presence + for the assigned condition. - Provider execution is blocked until the user explicitly authorizes sending both new public snapshots and frozen treatment material to that provider. - A technical interruption may be archived and restarted; a semantic outcome @@ -191,7 +203,7 @@ Pre-registered interpretation: `experiments/agent_qualification_generalization.py freeze` writes an immutable case manifest, seeded plan, and `freeze.json`. `validate` rejects drift in the tracked protocol, prompt, runner, skill, source snapshots, dependency seeds, -gold manifests, and evaluator evidence trees. +dependency destinations, gold manifests, and evaluator evidence trees. Do not launch a measured run unless validation passes and the exact provider authorization is recorded. diff --git a/experiments/agent_qualification.py b/experiments/agent_qualification.py index 6f44996..b249cdb 100644 --- a/experiments/agent_qualification.py +++ b/experiments/agent_qualification.py @@ -136,7 +136,10 @@ def _tree_inventory(root: Path) -> list[dict[str, Any]]: inventory: list[dict[str, Any]] = [] for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): - relative = path.relative_to(root).as_posix() + relative_path = path.relative_to(root) + if ".git" in relative_path.parts: + continue + relative = relative_path.as_posix() if path.is_symlink(): raise EvaluationError(f"frozen fixture must not contain symlink {relative!r}") mode = path.stat().st_mode @@ -870,6 +873,8 @@ def _claude_command(settings_path: Path) -> list[str]: "--no-session-persistence", "--no-chrome", "--disable-slash-commands", + "--setting-sources", + "", "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}', @@ -989,13 +994,9 @@ def _initialize_snapshot(source: Path, workspace: Path) -> None: _run_checked(("git", "commit", "-q", "-m", "Frozen candidate snapshot"), workspace) -def _write_workspace_evidence(workspace: Path, run_root: Path) -> None: - (run_root / "git-status.txt").write_text(_run_checked(("git", "status", "--short"), workspace)) - (run_root / "workspace.patch").write_text( - _run_checked(("git", "diff", "--binary", "HEAD"), workspace) - ) +def _workspace_inventory(workspace: Path) -> list[dict[str, Any]]: inventory: list[dict[str, Any]] = [] - excluded_parts = {".git", "bin", "obj", "target", "node_modules"} + excluded_parts = {".git", ".eval-cache", "bin", "obj", "target", "node_modules"} for path in sorted(workspace.rglob("*"), key=lambda value: value.as_posix()): relative = path.relative_to(workspace) if any(part in excluded_parts for part in relative.parts): @@ -1008,10 +1009,62 @@ def _write_workspace_evidence(workspace: Path, run_root: Path) -> None: "sha256": _sha256_file(path), } ) + return inventory + + +def _write_workspace_evidence(workspace: Path, run_root: Path) -> None: + """Capture tracked, untracked, and ignored candidate changes without mutating its index.""" + + inventory = _workspace_inventory(workspace) + temporary_index = run_root / ".workspace-evidence.index" + environment = dict(os.environ) + environment["GIT_INDEX_FILE"] = str(temporary_index) + + def run_git(*arguments: str) -> str: + completed = subprocess.run( + ("git", *arguments), + cwd=workspace, + env=environment, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + return completed.stdout + + try: + run_git("read-tree", "HEAD") + run_git("add", "-u", "--", ".") + paths = ["./" + str(item["path"]) for item in inventory] + for start in range(0, len(paths), 200): + run_git("add", "-f", "--", *paths[start : start + 200]) + (run_root / "git-status.txt").write_text(run_git("status", "--short")) + (run_root / "workspace.patch").write_text(run_git("diff", "--cached", "--binary", "HEAD")) + finally: + temporary_index.unlink(missing_ok=True) _write_json(run_root / "workspace-files.json", {"files": inventory}) -def execute_run( +def _quarantine_interrupted_run(run_root: Path, external_run_root: Path) -> Path | None: + """Preserve incomplete evidence under ``interruptions`` and unblock a clean retry.""" + + if external_run_root.is_symlink(): + external_run_root.unlink() + elif external_run_root.exists(): + shutil.rmtree(external_run_root) + if not run_root.exists() and not run_root.is_symlink(): + return None + if not run_root.is_symlink() and (run_root / "pre-score-verdict.json").is_file(): + return None + interruptions = run_root.parent / "interruptions" + interruptions.mkdir(parents=True, exist_ok=True) + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + target = interruptions / f"{run_root.name}.{stamp}.{os.getpid()}" + run_root.replace(target) + return target + + +def _execute_run_once( manifest_path: Path, *, case_id: str, @@ -1139,6 +1192,38 @@ def execute_run( return metadata +def execute_run( + manifest_path: Path, + *, + case_id: str, + agent: str, + repetition: int, + condition: str, + runs_path: Path, + timeout_seconds: int, +) -> dict[str, Any]: + run_id = f"{condition}__{case_id}__{agent}__r{repetition}" + run_root = runs_path.resolve() / run_id + work_root = CODEX_WORK_ROOT if agent == "codex" else CLAUDE_WORK_ROOT + external_run_root = work_root / run_id + run_root_preexisting = run_root.exists() or run_root.is_symlink() + external_root_preexisting = external_run_root.exists() or external_run_root.is_symlink() + try: + return _execute_run_once( + manifest_path, + case_id=case_id, + agent=agent, + repetition=repetition, + condition=condition, + runs_path=runs_path, + timeout_seconds=timeout_seconds, + ) + except BaseException: + if not run_root_preexisting and not external_root_preexisting: + _quarantine_interrupted_run(run_root, external_run_root) + raise + + def _parse_conditions(raw: str) -> list[str]: conditions = [item.strip() for item in raw.split(",") if item.strip()] unknown = [item for item in conditions if item not in {"baseline", "docs", "skill"}] diff --git a/experiments/agent_qualification_generalization.py b/experiments/agent_qualification_generalization.py index 9390453..1807364 100644 --- a/experiments/agent_qualification_generalization.py +++ b/experiments/agent_qualification_generalization.py @@ -26,18 +26,14 @@ PLAN_PATH = PRIVATE_ROOT / "plan.json" FREEZE_PATH = PRIVATE_ROOT / "freeze.json" MANIFESTS_ROOT = PRIVATE_ROOT / "manifests" -SKILL_PATH = ( - PRIVATE_ROOT - / "treatments" - / "tracebook-qualify-matching-engine" - / "SKILL.md" -) +SKILL_PATH = PRIVATE_ROOT / "treatments" / "tracebook-qualify-matching-engine" / "SKILL.md" PROTOCOL_ID = "agent-qualification-generalization-v2" RANDOMIZATION_SEED = 20260730 REPETITIONS = 3 AGENTS = ("codex", "claude") CONDITIONS = ("docs", "skill") +DEPENDENCY_TARGETS = ("m2-repository", "cargo-home") DOCS_TREATMENT = ( "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public " @@ -89,7 +85,10 @@ def _snapshot_digest(root: Path) -> str: def _tree_inventory(root: Path) -> list[dict[str, Any]]: inventory: list[dict[str, Any]] = [] for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): - relative = path.relative_to(root).as_posix() + relative_path = path.relative_to(root) + if ".git" in relative_path.parts: + continue + relative = relative_path.as_posix() if path.is_symlink(): raise GeneralizationError(f"frozen tree must not contain symlink {relative!r}") mode = path.stat().st_mode @@ -199,9 +198,7 @@ def _validate_tree( if not isinstance(declaration.get(field), str): raise GeneralizationError(f"case {case['id']!r} {name!r} lacks {field!r}") source = _private_path(str(declaration["source"]), f"{case['id']} {name} source") - manifest_path = _private_path( - str(declaration["manifest"]), f"{case['id']} {name} manifest" - ) + manifest_path = _private_path(str(declaration["manifest"]), f"{case['id']} {name} manifest") if not source.is_dir() or not manifest_path.is_file(): raise GeneralizationError(f"case {case['id']!r} {name!r} tree is unavailable") manifest_hash = _sha256_file(manifest_path) @@ -231,6 +228,14 @@ def _validate_tree( } +def _dependency_target(value: Any) -> str: + if value not in DEPENDENCY_TARGETS: + raise GeneralizationError( + "dependency_target must be one of " + ", ".join(DEPENDENCY_TARGETS) + ) + return str(value) + + def freeze() -> dict[str, Any]: """Create the immutable v2 case manifest, plan, and freeze binding.""" @@ -256,6 +261,7 @@ def freeze() -> dict[str, Any]: "origin", "gold_manifest", "dependency_source", + "dependency_target", "gold_evidence", "status", ) @@ -270,9 +276,7 @@ def freeze() -> dict[str, Any]: f"case {raw_case['id']!r} source must be an origin-stripped directory" ) snapshot_sha256 = _snapshot_digest(source) - gold_path = _private_path( - str(raw_case["gold_manifest"]), f"{raw_case['id']} gold manifest" - ) + gold_path = _private_path(str(raw_case["gold_manifest"]), f"{raw_case['id']} gold manifest") if not gold_path.is_file(): raise GeneralizationError(f"case {raw_case['id']!r} gold is unavailable") evidence_specs = raw_case["gold_evidence"] @@ -299,6 +303,7 @@ def freeze() -> dict[str, Any]: source_path=str(raw_case["dependency_source"]), snapshot_sha256=snapshot_sha256, ) + dependency["target"] = _dependency_target(raw_case["dependency_target"]) frozen_cases.append( { "id": raw_case["id"], @@ -407,6 +412,10 @@ def validate_manifest() -> dict[str, Any]: if _sha256_file(gold_path) != case["gold_sha256"]: raise GeneralizationError(f"case {case['id']!r} gold drifted") _validate_tree(case, case["dependency_cache"], name="dependency-cache") + dependency = case["dependency_cache"] + if not isinstance(dependency, dict): + raise GeneralizationError(f"case {case['id']!r} dependency cache is invalid") + _dependency_target(dependency.get("target")) evidence = case["gold_evidence"] if not isinstance(evidence, list): raise GeneralizationError(f"case {case['id']!r} gold_evidence must be a list") @@ -527,9 +536,7 @@ def main(argv: Sequence[str] | None = None) -> int: payload = validate() else: payload = validate() - payload["authorization_blocker"] = _load_json(FREEZE_PATH).get( - "execution_blocker" - ) + payload["authorization_blocker"] = _load_json(FREEZE_PATH).get("execution_blocker") print(json.dumps(payload, indent=2, sort_keys=True)) return 0 except GeneralizationError as exc: diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py index ade1d11..2d6f539 100644 --- a/experiments/agent_qualification_generalization_runs.py +++ b/experiments/agent_qualification_generalization_runs.py @@ -17,8 +17,9 @@ import socket import subprocess import sys +from contextlib import contextmanager from pathlib import Path -from typing import Any, Mapping, Sequence +from typing import Any, Iterator, Mapping, Sequence if __package__: from . import agent_qualification as helpers @@ -65,6 +66,15 @@ "CLAUDE_CODE_USE_VERTEX": "0", "CLAUDE_CODE_USE_FOUNDRY": "0", } +RESTRICTED_SYSTEM_PATH = ( + "/Library/Developer/CommandLineTools/usr/bin", + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", +) _PLUGIN_MANIFEST = { "author": {"name": "Tracebook evaluator"}, @@ -220,13 +230,50 @@ def _validated_provider_bindings() -> dict[str, dict[str, Any]]: return bindings -def _configure_delivery() -> None: - helpers.CODEX_BINARY = _provider_binary("codex") - helpers.CLAUDE_BINARY = _provider_binary("claude") - helpers.CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json" - delivery.DEFAULT_CLAUDE_WRAPPER_PATH = CLAUDE_WRAPPER_PATH - delivery.EXPECTED_MODELS = dict(EXPECTED_MODELS) - helpers._codex_shell_environment = _codex_shell_environment +@contextmanager +def _configured_delivery() -> Iterator[None]: + """Temporarily bind shared delivery helpers and always restore their globals.""" + + original_codex_binary = helpers.CODEX_BINARY + original_claude_binary = helpers.CLAUDE_BINARY + original_codex_auth_path = helpers.CODEX_AUTH_PATH + original_claude_wrapper_path = delivery.DEFAULT_CLAUDE_WRAPPER_PATH + original_expected_models = delivery.EXPECTED_MODELS + original_codex_shell_environment = helpers._codex_shell_environment + try: + helpers.CODEX_BINARY = _provider_binary("codex") + helpers.CLAUDE_BINARY = _provider_binary("claude") + helpers.CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json" + delivery.DEFAULT_CLAUDE_WRAPPER_PATH = CLAUDE_WRAPPER_PATH + delivery.EXPECTED_MODELS = dict(EXPECTED_MODELS) + helpers._codex_shell_environment = _codex_shell_environment + yield + finally: + helpers.CODEX_BINARY = original_codex_binary + helpers.CLAUDE_BINARY = original_claude_binary + helpers.CODEX_AUTH_PATH = original_codex_auth_path + delivery.DEFAULT_CLAUDE_WRAPPER_PATH = original_claude_wrapper_path + delivery.EXPECTED_MODELS = original_expected_models + helpers._codex_shell_environment = original_codex_shell_environment + + +def _toolchain_read_paths() -> tuple[Path, ...]: + paths: list[Path] = [] + for name in ( + "TRACEBOOK_V2_MAVEN_HOME", + "TRACEBOOK_V2_JAVA_HOME", + "TRACEBOOK_V2_PYTHON_HOME", + ): + raw = os.environ.get(name) + if raw: + paths.append(Path(raw).resolve()) + uv_binary = os.environ.get("TRACEBOOK_V2_UV_BINARY") + if uv_binary: + paths.append(Path(uv_binary).resolve().parent) + cargo_bin = Path.home() / ".cargo" / "bin" + if cargo_bin.is_dir(): + paths.append(cargo_bin.resolve()) + return tuple(dict.fromkeys(paths)) def _toolchain_environment() -> dict[str, str]: @@ -257,9 +304,7 @@ def _toolchain_environment() -> dict[str, str]: cargo_bin = Path.home() / ".cargo" / "bin" if cargo_bin.is_dir(): path_prefixes.append(str(cargo_bin)) - values["PATH"] = ":".join( - [*path_prefixes, os.environ.get("PATH", "/usr/bin:/bin:/usr/sbin:/sbin")] - ) + values["PATH"] = ":".join(dict.fromkeys([*path_prefixes, *RESTRICTED_SYSTEM_PATH])) return values @@ -275,6 +320,10 @@ def _claude_settings(workspace: Path, scratch: Path, plugin_root: Path) -> dict[ settings["env"].update(CLAUDE_FIRST_PARTY_ENV) settings["env"].update(_toolchain_environment()) settings["env"].update(_fresh_environment(scratch)) + allow_read = settings["sandbox"]["filesystem"]["allowRead"] + for path in _toolchain_read_paths(): + if str(path) not in allow_read: + allow_read.append(str(path)) return settings @@ -434,22 +483,37 @@ def _native_surface( skill: bytes, skill_name: str, ) -> tuple[list[str], dict[str, Any], Path | None]: - _configure_delivery() - return delivery._prepare_native_surface( - agent=provider, - condition=condition, - workspace=workspace, - external_run_root=run_root, - scratch=scratch, - environment=environment, - skill_content=skill, - skill_name=skill_name, - ) + with _configured_delivery(): + command, surface, plugin_root = delivery._prepare_native_surface( + agent=provider, + condition=condition, + workspace=workspace, + external_run_root=run_root, + scratch=scratch, + environment=environment, + skill_content=skill, + skill_name=skill_name, + ) + if provider == "codex": + permission_index = next( + index + for index, value in enumerate(command) + if value.startswith("permissions.agent-eval=") + ) + read_entries = "".join( + f'{json.dumps(str(path))}="read",' for path in _toolchain_read_paths() + ) + command[permission_index] = command[permission_index].replace( + "filesystem={", + "filesystem={" + read_entries, + 1, + ) + return command, surface, plugin_root def _claude_command(settings_path: Path, plugin_root: Path) -> list[str]: - command = delivery._claude_treatment_command(settings_path, plugin_root) - return command + with _configured_delivery(): + return delivery._claude_treatment_command(settings_path, plugin_root) def _run_delivery_probe(provider: str, condition: str, root: Path) -> dict[str, Any]: @@ -502,12 +566,21 @@ def _run_delivery_probe(provider: str, condition: str, root: Path) -> dict[str, ) terminal = delivery._transcript_terminal_state(transcript, provider) result = delivery._transcript_result_text(transcript, provider) + catalog_audit = None + if provider == "claude": + catalog_audit = delivery._claude_catalog_audit( + transcript, + condition=condition, + skill_name=SYNTHETIC_SKILL_NAME, + ) expected = SYNTHETIC_MARKER if condition == "skill" else "NO_NATIVE_DELIVERY" errors = [] if exit_code != 0 or timed_out or terminal != "completed": errors.append("provider did not complete cleanly") if audit.get("mutated") is not False: errors.append("native skill surface mutated") + if provider == "claude" and catalog_audit is not None and catalog_audit["valid"] is not True: + errors.append("Claude provider catalog isolation failed") if result != expected: errors.append(f"result {result!r} != {expected!r}") return { @@ -520,6 +593,7 @@ def _run_delivery_probe(provider: str, condition: str, root: Path) -> dict[str, "terminal_state": terminal, "result": result, "surface": audit, + "provider_catalog_audit": catalog_audit, "transcript": _file_record(transcript), "stderr": _file_record(stderr), } @@ -666,7 +740,7 @@ def _copy_fixture(case: Mapping[str, Any], scratch: Path) -> dict[str, Any]: if not isinstance(declaration, dict): raise ExecutionError("case dependency-cache declaration is missing") source = (REPOSITORY_ROOT / str(declaration["source"])).resolve(strict=True) - target_name = "m2-repository" if case["id"] == "c4-compatible" else "cargo-home" + target_name = cohort._dependency_target(declaration.get("target")) target = scratch / target_name helpers._copy_frozen_tree(source, target, str(declaration["tree_sha256"])) return { @@ -680,9 +754,17 @@ def _record_fixture_final(fixture: dict[str, Any]) -> None: target = Path(str(fixture["target"])) if not target.is_dir(): fixture["final_state"] = "missing" + fixture["mutated"] = True + return + try: + final_hash = helpers._snapshot_digest(target) + except OSError as exc: + fixture["final_state"] = f"unreadable:{type(exc).__name__}" + fixture["mutated"] = True return fixture["final_state"] = "present" - fixture["final_tree_sha256"] = helpers._snapshot_digest(target) + fixture["final_tree_sha256"] = final_hash + fixture["mutated"] = final_hash != fixture["initial_tree_sha256"] def _run_root(run_id: str) -> Path: @@ -713,9 +795,20 @@ def _technical_verdict( return verdict -def validate_run(run_id: str) -> dict[str, Any]: - freeze = validate_freeze() - manifest, plan, _ = _load_primary() +def validate_run( + run_id: str, + *, + freeze: Mapping[str, Any] | None = None, + primary: tuple[Mapping[str, Any], Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + if freeze is None: + freeze = validate_freeze() + if primary is None: + loaded_manifest, loaded_plan, _ = _load_primary() + manifest: Mapping[str, Any] = loaded_manifest + plan: Mapping[str, Any] = loaded_plan + else: + manifest, plan = primary entry = _plan_entry(plan, run_id) run_root = _run_root(run_id) metadata = _read_json(run_root / "metadata.json", "run metadata") @@ -793,6 +886,18 @@ def validate_run(run_id: str) -> dict[str, Any]: or surface["final_audit"].get("mutated") is not False ): errors.append("native skill surface audit failed") + if entry["agent"] == "claude": + catalog_audit = surface.get("provider_catalog_audit") + if not isinstance(catalog_audit, dict) or catalog_audit.get("valid") is not True: + errors.append("Claude provider catalog audit failed") + fixture = metadata.get("fixture") + if not isinstance(fixture, dict): + errors.append("dependency fixture audit is missing") + else: + if fixture.get("mutated") is not False: + errors.append("dependency fixture mutated") + if fixture.get("final_tree_sha256") != fixture.get("initial_tree_sha256"): + errors.append("dependency fixture final hash changed") for artifact_name in ( "git-status.txt", "workspace.patch", @@ -814,17 +919,28 @@ def validate_run(run_id: str) -> dict[str, Any]: return verdict -def _validate_prior_runs(plan: Mapping[str, Any], order: int) -> None: +def _validate_prior_runs( + plan: Mapping[str, Any], + order: int, + *, + freeze: Mapping[str, Any] | None = None, + primary: tuple[Mapping[str, Any], Mapping[str, Any]] | None = None, +) -> None: entries = plan.get("entries") if not isinstance(entries, list): raise ExecutionError("validated cohort lost its plan entries") for prior in entries[: order - 1]: if not isinstance(prior, dict): raise ExecutionError("plan entry shape changed") - validate_run(str(prior["id"])) + if freeze is None or primary is None: + validate_run(str(prior["id"])) + else: + validate_run(str(prior["id"]), freeze=freeze, primary=primary) -def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> dict[str, Any]: +def _execute_run_once( + run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS +) -> dict[str, Any]: if timeout_seconds != DEFAULT_TIMEOUT_SECONDS: raise ExecutionError(f"official timeout must remain {DEFAULT_TIMEOUT_SECONDS} seconds") freeze = validate_freeze() @@ -832,7 +948,12 @@ def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> entry = _plan_entry(plan, run_id) provider = str(entry["agent"]) validate_authorization(provider) - _validate_prior_runs(plan, int(entry["order"])) + _validate_prior_runs( + plan, + int(entry["order"]), + freeze=freeze, + primary=(manifest, plan), + ) case = _case_index(manifest)[str(entry["case_id"])] run_root = _run_root(run_id) external_root = _external_run_root(run_id) @@ -939,6 +1060,12 @@ def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> ignore=shutil.ignore_patterns(".git", "target", "node_modules", ".eval-cache"), ) terminal = delivery._transcript_terminal_state(transcript, provider) + if provider == "claude": + surface["provider_catalog_audit"] = delivery._claude_catalog_audit( + transcript, + condition=str(entry["condition"]), + skill_name=SKILL_NAME, + ) model, model_source = helpers._reported_model(transcript, provider) metadata.update( { @@ -967,6 +1094,10 @@ def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> errors.append(f"reported model {model!r} != {EXPECTED_MODELS[provider]!r}") if surface["final_audit"].get("mutated") is not False: errors.append("native skill surface mutated") + if provider == "claude" and surface.get("provider_catalog_audit", {}).get("valid") is not True: + errors.append("Claude provider catalog isolation failed") + if fixture.get("mutated") is not False: + errors.append("dependency fixture mutated") verdict = _technical_verdict(run_root, metadata, errors) if errors: raise ExecutionError(f"run {run_id!r} is technically invalid: " + "; ".join(errors)) @@ -975,6 +1106,19 @@ def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> return {"metadata": metadata, "verdict": verdict} +def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> dict[str, Any]: + run_root = _run_root(run_id) + external_root = _external_run_root(run_id) + run_root_preexisting = run_root.exists() or run_root.is_symlink() + external_root_preexisting = external_root.exists() or external_root.is_symlink() + try: + return _execute_run_once(run_id, timeout_seconds) + except BaseException: + if not run_root_preexisting and not external_root_preexisting: + helpers._quarantine_interrupted_run(run_root, external_root) + raise + + def status() -> dict[str, Any]: cohort_summary = cohort.validate() _, plan, _ = _load_primary() diff --git a/experiments/agent_qualification_treatments.py b/experiments/agent_qualification_treatments.py index b023805..a54cd92 100644 --- a/experiments/agent_qualification_treatments.py +++ b/experiments/agent_qualification_treatments.py @@ -12,6 +12,8 @@ import json import os import random +import re +import secrets import shlex import shutil import stat @@ -293,11 +295,11 @@ def _load_skill(skill_path: Path, manifest: Mapping[str, Any]) -> dict[str, Any] def _ready_case_ids(manifest: Mapping[str, Any]) -> list[str]: - return sorted( + return [ case_id for case_id, case in baseline._case_index(manifest).items() if case.get("status") == "ready" - ) + ] def _structural_cells(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: @@ -319,13 +321,25 @@ def _structural_cells(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: return cells -def _evidence_id(ordinal: int) -> str: - material = f"{PROTOCOL_ID}|treatment|{RANDOMIZATION_SEED}|{ordinal}".encode() - return "ev_" + _sha256_bytes(material)[:20] +def _new_evidence_id() -> str: + """Create a non-derivable identifier for blinded pre-score evidence.""" + return "ev_" + secrets.token_hex(16) -def _master_entries(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: + +def _master_entries( + manifest: Mapping[str, Any], + *, + evidence_ids: Sequence[str] | None = None, +) -> list[dict[str, Any]]: cells = _structural_cells(manifest) + expected_count = len(TREATMENT_CONDITIONS) * len(cells) + if evidence_ids is None: + evidence_ids = [_new_evidence_id() for _ in range(expected_count)] + if len(evidence_ids) != expected_count: + raise EvaluationError( + f"treatment protocol requires {expected_count} evidence IDs, got {len(evidence_ids)}" + ) entries: list[dict[str, Any]] = [] ordinal = 0 for condition in TREATMENT_CONDITIONS: @@ -337,7 +351,7 @@ def _master_entries(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: **cell, "condition": condition, "run_id": run_id, - "evidence_id": _evidence_id(ordinal), + "evidence_id": evidence_ids[ordinal - 1], "ordinal": ordinal, } ) @@ -545,7 +559,11 @@ def _validate_plan_entry_shape(entries: Any) -> list[dict[str, Any]]: evidence_id = entry.get("evidence_id") if not isinstance(run_id, str) or run_id in seen_ids: raise EvaluationError("master plan run IDs must be unique strings") - if not isinstance(evidence_id, str) or evidence_id in seen_evidence: + if ( + not isinstance(evidence_id, str) + or re.fullmatch(r"ev_[0-9a-f]{32}", evidence_id) is None + or evidence_id in seen_evidence + ): raise EvaluationError("master plan evidence IDs must be unique opaque strings") if entry.get("condition") not in TREATMENT_CONDITIONS: raise EvaluationError("master plan contains an invalid condition") @@ -623,7 +641,10 @@ def validate_plan( raise EvaluationError("master plan rendered prompt hashes changed") entries = _validate_plan_entry_shape(payload.get("entries")) - expected_entries = _master_entries(manifest) + expected_entries = _master_entries( + manifest, + evidence_ids=[str(entry["evidence_id"]) for entry in entries], + ) prompt_hashes = _prompt_hashes(manifest) for entry in expected_entries: entry["rendered_prompt_sha256"] = prompt_hashes[entry["case_id"]][entry["condition"]] @@ -1061,7 +1082,10 @@ def _claude_catalog_audit( init: dict[str, Any] | None = None with path.open(encoding="utf-8") as handle: for line in handle: - event = json.loads(line) + try: + event = json.loads(line) + except json.JSONDecodeError: + continue if ( isinstance(event, dict) and event.get("type") == "system" @@ -1316,7 +1340,7 @@ def validate_run( return verdict -def execute_run( +def _execute_run_once( run_id: str, *, timeout_seconds: int = baseline.DEFAULT_TIMEOUT_SECONDS, @@ -1480,11 +1504,34 @@ def execute_run( return {"metadata": metadata, "verdict": verdict} +def execute_run( + run_id: str, + *, + timeout_seconds: int = baseline.DEFAULT_TIMEOUT_SECONDS, +) -> dict[str, Any]: + run_root = _run_root(run_id) + parts = run_id.split("__") + agent = parts[-2] if len(parts) >= 4 else "" + work_root = baseline.CODEX_WORK_ROOT if agent == "codex" else baseline.CLAUDE_WORK_ROOT + external_run_root = work_root / run_id + run_root_preexisting = run_root.exists() or run_root.is_symlink() + external_root_preexisting = external_run_root.exists() or external_run_root.is_symlink() + try: + return _execute_run_once(run_id, timeout_seconds=timeout_seconds) + except BaseException: + if not run_root_preexisting and not external_root_preexisting: + baseline._quarantine_interrupted_run(run_root, external_run_root) + raise + + def _transcript_result_text(path: Path, agent: str) -> str: result = "" with path.open(encoding="utf-8") as handle: for line in handle: - event = json.loads(line) + try: + event = json.loads(line) + except json.JSONDecodeError: + continue if not isinstance(event, dict): continue if agent == "codex" and event.get("type") == "item.completed": @@ -1676,7 +1723,7 @@ def main(argv: Sequence[str] | None = None) -> int: payload = validate_run(args.run_id) else: # pragma: no cover - argparse enforces the command. raise EvaluationError(f"unknown command {args.command!r}") - except (EvaluationError, OSError, ValueError) as exc: + except (EvaluationError, OSError, ValueError, subprocess.SubprocessError) as exc: print(f"error: {exc}", file=sys.stderr) return 2 printable = { diff --git a/tests/test_agent_qualification_experiment.py b/tests/test_agent_qualification_experiment.py index bf3564c..24e2e4a 100644 --- a/tests/test_agent_qualification_experiment.py +++ b/tests/test_agent_qualification_experiment.py @@ -91,6 +91,19 @@ def test_validate_manifest_rejects_snapshot_drift(tmp_path): agent_qualification.validate_manifest(manifest_path) +def test_inventory_and_digest_both_ignore_git_metadata(tmp_path): + source = tmp_path / "source" + git = source / ".git" + git.mkdir(parents=True) + (source / "engine.txt").write_text("price-time\n") + (git / "index").write_text("first\n") + first_digest = agent_qualification._snapshot_digest(source) + + assert [item["path"] for item in agent_qualification._tree_inventory(source)] == ["engine.txt"] + (git / "index").write_text("changed\n") + assert agent_qualification._snapshot_digest(source) == first_digest + + def test_validate_manifest_rejects_gold_drift(tmp_path): manifest_path = _manifest(tmp_path) manifest = agent_qualification.validate_manifest(manifest_path) @@ -245,6 +258,13 @@ def test_claude_permissions_delegate_paths_to_fail_closed_sandbox(tmp_path): assert settings["env"]["NuGetAudit"] == "false" +def test_claude_command_ignores_user_and_project_settings(tmp_path): + command = agent_qualification._claude_command(tmp_path / "settings.json") + + assert command[command.index("--setting-sources") + 1] == "" + assert command[command.index("--settings") + 1] == str(tmp_path / "settings.json") + + def test_dotnet_environment_preseeds_nuget_migration_state(tmp_path): environment = agent_qualification._dotnet_environment(tmp_path) @@ -342,3 +362,43 @@ def test_reported_model_uses_claude_transcript_and_codex_command(tmp_path): "gpt-5.6-sol", "command", ) + + +def test_workspace_evidence_includes_gitignored_files(tmp_path): + source = tmp_path / "source" + source.mkdir() + (source / ".gitignore").write_text("ignored.txt\n") + (source / "tracked.txt").write_text("before\n") + workspace = tmp_path / "workspace" + agent_qualification._initialize_snapshot(source, workspace) + (workspace / "tracked.txt").write_text("after\n") + (workspace / "ignored.txt").write_text("captured\n") + run_root = tmp_path / "run" + run_root.mkdir() + + agent_qualification._write_workspace_evidence(workspace, run_root) + + status = (run_root / "git-status.txt").read_text() + patch = (run_root / "workspace.patch").read_text() + inventory = json.loads((run_root / "workspace-files.json").read_text())["files"] + assert "ignored.txt" in status + assert "ignored.txt" in patch + assert "captured" in patch + assert "ignored.txt" in {item["path"] for item in inventory} + + +def test_interrupted_run_is_quarantined_and_external_workspace_removed(tmp_path): + run_root = tmp_path / "runs" / "docs__case__codex__r1" + external = tmp_path / "external" + run_root.mkdir(parents=True) + external.mkdir() + (run_root / "partial.jsonl").write_text("partial\n") + (external / "workspace").mkdir() + + archived = agent_qualification._quarantine_interrupted_run(run_root, external) + + assert archived is not None + assert archived.parent.name == "interruptions" + assert (archived / "partial.jsonl").read_text() == "partial\n" + assert not run_root.exists() + assert not external.exists() diff --git a/tests/test_agent_qualification_generalization.py b/tests/test_agent_qualification_generalization.py index 8aed573..b5c1cb3 100644 --- a/tests/test_agent_qualification_generalization.py +++ b/tests/test_agent_qualification_generalization.py @@ -92,6 +92,24 @@ def test_snapshot_digest_binds_file_mode_and_content(tmp_path): assert len({first, second, third}) == 3 +def test_tree_inventory_uses_same_git_exclusion_as_snapshot_digest(tmp_path): + source = tmp_path / "source" + git = source / ".git" + git.mkdir(parents=True) + (source / "engine.txt").write_text("price-time\n") + (git / "index").write_text("metadata\n") + + assert [item["path"] for item in generalization._tree_inventory(source)] == ["engine.txt"] + + +def test_dependency_target_is_explicit_and_restricted(): + assert generalization._dependency_target("m2-repository") == "m2-repository" + assert generalization._dependency_target("cargo-home") == "cargo-home" + + with pytest.raises(generalization.GeneralizationError, match="dependency_target"): + generalization._dependency_target("../operator-cache") + + def test_validate_tree_rejects_inventory_drift(tmp_path, monkeypatch): private = tmp_path / "private" source = private / "cache" diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py index fc9883a..2382f5a 100644 --- a/tests/test_agent_qualification_generalization_runs.py +++ b/tests/test_agent_qualification_generalization_runs.py @@ -1,5 +1,6 @@ import hashlib import json +from contextlib import nullcontext from pathlib import Path import pytest @@ -84,6 +85,7 @@ def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch): "source": "private/cache", "tree_sha256": tree_hash, "manifest_sha256": "manifest", + "target": "cargo-home", }, } @@ -93,6 +95,11 @@ def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch): assert Path(fixture["target"]).name == "cargo-home" assert execution.helpers._snapshot_digest(Path(fixture["target"])) == tree_hash + (Path(fixture["target"]) / "artifact").write_text("mutated") + execution._record_fixture_final(fixture) + assert fixture["mutated"] is True + assert fixture["final_tree_sha256"] != fixture["initial_tree_sha256"] + def test_native_surface_has_no_skill_in_docs_condition(tmp_path, monkeypatch): root = tmp_path / "run" @@ -102,14 +109,14 @@ def test_native_surface_has_no_skill_in_docs_condition(tmp_path, monkeypatch): scratch.mkdir() monkeypatch.setattr( execution, - "_configure_delivery", - lambda: None, + "_configured_delivery", + lambda: nullcontext(), ) monkeypatch.setattr( execution.delivery, "_prepare_native_surface", lambda **kwargs: ( - ["provider"], + ["provider", "permissions.agent-eval={filesystem={}}"], { "skill_installed": kwargs["condition"] == "skill", "semantic_file_count": 1 if kwargs["condition"] == "skill" else 0, @@ -132,6 +139,45 @@ def test_native_surface_has_no_skill_in_docs_condition(tmp_path, monkeypatch): assert surface == {"skill_installed": False, "semantic_file_count": 0} +def test_codex_native_surface_can_read_declared_toolchain_roots(tmp_path, monkeypatch): + root = tmp_path / "run" + workspace = root / "workspace" + scratch = root / "scratch" + maven = tmp_path / "maven" + java = tmp_path / "java" + workspace.mkdir(parents=True) + scratch.mkdir() + (maven / "bin").mkdir(parents=True) + (java / "bin").mkdir(parents=True) + monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven)) + monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java)) + monkeypatch.setattr(execution, "_configured_delivery", lambda: nullcontext()) + monkeypatch.setattr( + execution.delivery, + "_prepare_native_surface", + lambda **kwargs: ( + ["provider", "permissions.agent-eval={filesystem={}}"], + {"skill_installed": False}, + None, + ), + ) + + command, _, _ = execution._native_surface( + provider="codex", + condition="docs", + workspace=workspace, + run_root=root, + scratch=scratch, + environment={}, + skill=b"skill", + skill_name="probe", + ) + + permission = next(value for value in command if value.startswith("permissions.agent-eval=")) + assert f'"{maven}"="read"' in permission + assert f'"{java}"="read"' in permission + + def test_validate_prior_runs_enforces_seeded_order(monkeypatch): seen = [] monkeypatch.setattr(execution, "validate_run", seen.append) @@ -148,6 +194,22 @@ def test_validate_prior_runs_enforces_seeded_order(monkeypatch): assert seen == ["first", "second"] +def test_validate_prior_runs_reuses_one_frozen_context(monkeypatch): + seen = [] + freeze = {"frozen": True} + primary = ({"cases": []}, {"entries": []}) + + def validate(run_id, **kwargs): + seen.append((run_id, kwargs["freeze"], kwargs["primary"])) + + monkeypatch.setattr(execution, "validate_run", validate) + plan = {"entries": [{"id": "first"}, {"id": "second"}, {"id": "third"}]} + + execution._validate_prior_runs(plan, 3, freeze=freeze, primary=primary) + + assert seen == [("first", freeze, primary), ("second", freeze, primary)] + + def test_execution_refuses_without_provider_authorization(monkeypatch): monkeypatch.setattr(execution, "validate_freeze", lambda: {}) monkeypatch.setattr( @@ -206,7 +268,7 @@ def test_toolchain_environment_prepends_frozen_tools(tmp_path, monkeypatch): monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java)) monkeypatch.setenv("TRACEBOOK_V2_PYTHON_HOME", str(python)) monkeypatch.setenv("TRACEBOOK_V2_UV_BINARY", str(uv)) - monkeypatch.setenv("PATH", "/usr/bin:/bin") + monkeypatch.setenv("PATH", "/operator/private/bin") environment = execution._toolchain_environment() @@ -217,6 +279,8 @@ def test_toolchain_environment_prepends_frozen_tools(tmp_path, monkeypatch): str(python / "bin"), str(tmp_path), ] + assert "/operator/private/bin" not in environment["PATH"] + assert "/usr/bin" in environment["PATH"].split(":") def test_provider_shells_receive_fresh_paths(tmp_path, monkeypatch): @@ -266,18 +330,75 @@ def run(command, **kwargs): assert binding["auth_route"]["apiProvider"] == "firstParty" -def test_claude_run_settings_disable_third_party_routes(tmp_path): +def test_claude_run_settings_disable_third_party_routes(tmp_path, monkeypatch): workspace = tmp_path / "workspace" scratch = tmp_path / "scratch" plugin_root = tmp_path / "plugin" workspace.mkdir() scratch.mkdir() plugin_root.mkdir() + maven = tmp_path / "maven" + java = tmp_path / "java" + (maven / "bin").mkdir(parents=True) + (java / "bin").mkdir(parents=True) + monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven)) + monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java)) settings = execution._claude_settings(workspace, scratch, plugin_root) for name, value in execution.CLAUDE_FIRST_PARTY_ENV.items(): assert settings["env"][name] == value + assert str(maven) in settings["sandbox"]["filesystem"]["allowRead"] + assert str(java) in settings["sandbox"]["filesystem"]["allowRead"] + + +def test_delivery_configuration_restores_imported_module_globals(tmp_path, monkeypatch): + binary = tmp_path / "provider" + binary.write_text("provider") + monkeypatch.setattr(execution, "_provider_binary", lambda provider: binary) + original = ( + execution.helpers.CODEX_BINARY, + execution.helpers.CLAUDE_BINARY, + execution.helpers.CODEX_AUTH_PATH, + execution.delivery.DEFAULT_CLAUDE_WRAPPER_PATH, + execution.delivery.EXPECTED_MODELS, + execution.helpers._codex_shell_environment, + ) + + with execution._configured_delivery(): + assert execution.helpers.CODEX_BINARY == binary + assert execution.helpers.CLAUDE_BINARY == binary + assert execution.helpers._codex_shell_environment is execution._codex_shell_environment + + assert ( + execution.helpers.CODEX_BINARY, + execution.helpers.CLAUDE_BINARY, + execution.helpers.CODEX_AUTH_PATH, + execution.delivery.DEFAULT_CLAUDE_WRAPPER_PATH, + execution.delivery.EXPECTED_MODELS, + execution.helpers._codex_shell_environment, + ) == original + + +def test_interrupted_v2_run_is_quarantined_for_clean_restart(tmp_path, monkeypatch): + runs = tmp_path / "runs" + external = tmp_path / "external" + monkeypatch.setattr(execution, "RUNS_ROOT", runs) + monkeypatch.setattr(execution, "EXTERNAL_ROOT", external) + + def fail(run_id, timeout_seconds): + (runs / run_id).mkdir(parents=True) + (external / run_id / "workspace").mkdir(parents=True) + raise KeyboardInterrupt("interrupted") + + monkeypatch.setattr(execution, "_execute_run_once", fail) + + with pytest.raises(KeyboardInterrupt, match="interrupted"): + execution.execute_run("docs__case__codex__r1") + + assert not (runs / "docs__case__codex__r1").exists() + assert not (external / "docs__case__codex__r1").exists() + assert len(list((runs / "interruptions").iterdir())) == 1 def test_shakedown_requires_first_party_claude_route(monkeypatch): diff --git a/tests/test_agent_qualification_treatments.py b/tests/test_agent_qualification_treatments.py index 78ca1f6..4580efd 100644 --- a/tests/test_agent_qualification_treatments.py +++ b/tests/test_agent_qualification_treatments.py @@ -10,7 +10,7 @@ from experiments import agent_qualification from experiments import agent_qualification_treatments as treatments -BASELINE_RUNNER_SHA256 = "6ccf1f0ceee34672e049c428cb368d98a1c36270aeff21900f25b79e659261ef" +BASELINE_RUNNER_SHA256 = "71300722b273e983dcd4e830358677337255a39f52b7f10e4c57479c8dc5cb0d" def _sha256(path: Path) -> str: @@ -51,7 +51,8 @@ def test_treatment_prompt_is_identical_between_docs_and_skill(): def test_master_plan_is_two_matched_blocks_in_baseline_order(): manifest = _synthetic_manifest() - entries = treatments._master_entries(manifest) + evidence_ids = [f"ev_{index:032x}" for index in range(24)] + entries = treatments._master_entries(manifest, evidence_ids=evidence_ids) baseline_cells = [ { "case_id": case["id"], @@ -78,6 +79,43 @@ def test_master_plan_is_two_matched_blocks_in_baseline_order(): ] == baseline_cells assert len({entry["run_id"] for entry in entries}) == 24 assert len({entry["evidence_id"] for entry in entries}) == 24 + assert [entry["evidence_id"] for entry in entries] == evidence_ids + + +def test_master_plan_preserves_manifest_case_order_before_seeded_shuffle(): + manifest = { + "cases": [ + {"id": "z-last-lexically", "status": "ready"}, + {"id": "a-first-lexically", "status": "ready"}, + ] + } + evidence_ids = [f"ev_{index:032x}" for index in range(24)] + entries = treatments._master_entries(manifest, evidence_ids=evidence_ids) + expected = [ + {"case_id": case["id"], "agent": agent, "repetition": repetition} + for case in manifest["cases"] + for agent in ("codex", "claude") + for repetition in range(1, treatments.REPETITIONS + 1) + ] + random.Random(agent_qualification.RANDOMIZATION_SEED).shuffle(expected) + + assert [ + { + "case_id": entry["case_id"], + "agent": entry["agent"], + "repetition": entry["repetition"], + } + for entry in entries[:12] + ] == expected + + +def test_evidence_ids_are_random_opaque_values(): + first = treatments._new_evidence_id() + second = treatments._new_evidence_id() + + assert first != second + assert len(first) == len("ev_") + 32 + assert int(first.removeprefix("ev_"), 16) >= 0 def test_skill_is_candidate_agnostic_and_exactly_one_measured_file(tmp_path, monkeypatch): @@ -133,7 +171,7 @@ def test_master_plan_shape_rejects_tampering(): { "ordinal": index, "run_id": f"run-{index}", - "evidence_id": f"evidence-{index}", + "evidence_id": f"ev_{index:032x}", "condition": "docs" if index <= 12 else "skill", } for index in range(1, 25) @@ -284,7 +322,8 @@ def test_transcript_terminal_state_requires_clean_completion(tmp_path): def test_claude_catalog_audit_requires_only_wrapper_and_expected_skill(tmp_path): transcript = tmp_path / "claude.jsonl" transcript.write_text( - json.dumps( + "provider banner that is not JSON\n" + + json.dumps( { "type": "system", "subtype": "init", @@ -310,6 +349,14 @@ def test_claude_catalog_audit_requires_only_wrapper_and_expected_skill(tmp_path) assert treatments._claude_catalog_audit(transcript, condition="skill")["valid"] is True assert treatments._claude_catalog_audit(transcript, condition="docs")["valid"] is False + assert treatments._transcript_result_text(transcript, "claude") == "" + + +def test_transcript_result_text_skips_non_json_lines(tmp_path): + transcript = tmp_path / "claude-result.jsonl" + transcript.write_text('banner\n{"type":"result","result":"done"}\n') + + assert treatments._transcript_result_text(transcript, "claude") == "done" def test_measurement_policy_uses_conservative_paired_timing_gate(): @@ -326,6 +373,17 @@ def test_official_run_rejects_timeout_drift_before_freeze_validation(): treatments.execute_run("not-a-run", timeout_seconds=1) +def test_main_converts_subprocess_failures_to_exit_two(monkeypatch, capsys): + monkeypatch.setattr( + treatments, + "run_native_delivery_shakedown", + lambda: (_ for _ in ()).throw(subprocess.CalledProcessError(1, ["git"])), + ) + + assert treatments.main(["shakedown"]) == 2 + assert "error:" in capsys.readouterr().err + + def test_direct_script_entrypoint_smoke(): completed = subprocess.run( [ From 6a7f71df7bc116386e769f63ab0e6e52a329273c Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Tue, 25 Aug 2026 03:55:11 -0400 Subject: [PATCH 06/19] deduplicate Codex experiment permissions --- experiments/agent_qualification_generalization_runs.py | 7 +++++-- tests/test_agent_qualification_generalization_runs.py | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py index 2d6f539..8a0f34f 100644 --- a/experiments/agent_qualification_generalization_runs.py +++ b/experiments/agent_qualification_generalization_runs.py @@ -500,10 +500,13 @@ def _native_surface( for index, value in enumerate(command) if value.startswith("permissions.agent-eval=") ) + permission_profile = command[permission_index] read_entries = "".join( - f'{json.dumps(str(path))}="read",' for path in _toolchain_read_paths() + f'{json.dumps(str(path))}="read",' + for path in _toolchain_read_paths() + if f"{json.dumps(str(path))}=" not in permission_profile ) - command[permission_index] = command[permission_index].replace( + command[permission_index] = permission_profile.replace( "filesystem={", "filesystem={" + read_entries, 1, diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py index 2382f5a..ede6675 100644 --- a/tests/test_agent_qualification_generalization_runs.py +++ b/tests/test_agent_qualification_generalization_runs.py @@ -149,6 +149,8 @@ def test_codex_native_surface_can_read_declared_toolchain_roots(tmp_path, monkey scratch.mkdir() (maven / "bin").mkdir(parents=True) (java / "bin").mkdir(parents=True) + cargo_bin = Path.home() / ".cargo" / "bin" + existing_cargo_read = f'{json.dumps(str(cargo_bin))}="read",' monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven)) monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java)) monkeypatch.setattr(execution, "_configured_delivery", lambda: nullcontext()) @@ -156,7 +158,10 @@ def test_codex_native_surface_can_read_declared_toolchain_roots(tmp_path, monkey execution.delivery, "_prepare_native_surface", lambda **kwargs: ( - ["provider", "permissions.agent-eval={filesystem={}}"], + [ + "provider", + f"permissions.agent-eval={{filesystem={{{existing_cargo_read}}}}}", + ], {"skill_installed": False}, None, ), @@ -176,6 +181,7 @@ def test_codex_native_surface_can_read_declared_toolchain_roots(tmp_path, monkey permission = next(value for value in command if value.startswith("permissions.agent-eval=")) assert f'"{maven}"="read"' in permission assert f'"{java}"="read"' in permission + assert permission.count(existing_cargo_read) == 1 def test_validate_prior_runs_enforces_seeded_order(monkeypatch): From 6d266d6a6e119db3bbbc10ee1c7864875583b16b Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Wed, 26 Aug 2026 04:51:26 -0400 Subject: [PATCH 07/19] isolate Cargo fixture from mutable tool state --- ...t-qualification-generalization-execution-v2.md | 15 +++++++++------ docs/agent-qualification-generalization-v2.md | 5 +++-- experiments/agent_qualification_generalization.py | 2 +- .../agent_qualification_generalization_runs.py | 12 ++++++++++++ tests/test_agent_qualification_generalization.py | 1 + ...est_agent_qualification_generalization_runs.py | 12 ++++++++++-- 6 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md index 720677e..e6df7a7 100644 --- a/docs/agent-qualification-generalization-execution-v2.md +++ b/docs/agent-qualification-generalization-execution-v2.md @@ -42,10 +42,12 @@ for the earlier C1/C3 cohort does not apply. ## Isolation And Order -Every run receives a new origin-stripped workspace, new writable dependency -seed, new build target, isolated provider state, and condition-specific native -skill surface. The local Tracebook checkout, private gold, prior workspaces, -and prior transcripts are not mounted into the workspace. +Every run receives a new origin-stripped workspace, new dependency-seed copy, +new writable tool state and build target, isolated provider state, and +condition-specific native skill surface. For Cargo cases, the hashed fixture is +an immutable vendored-source tree while Cargo home and target directories are +separate fresh writable paths. The local Tracebook checkout, private gold, +prior workspaces, and prior transcripts are not mounted into the workspace. Before accepting a Claude turn, the harness audits the provider catalog and requires no MCP servers, exactly one intended plugin, and skill visibility that @@ -63,8 +65,9 @@ reported model before semantic grading. A pre-score verdict covers technical validity only; it must not inspect whether the agent reached the correct qualification outcome. -Fixture validity requires the final dependency-cache hash to equal its frozen -initial hash. Workspace inventory and patch capture include ignored candidate +Fixture validity requires the final dependency-seed hash to equal its frozen +initial hash. Tool-generated usage metadata lives outside that seed. Workspace +inventory and patch capture include ignored candidate files, using a temporary Git index so the evaluator's real index is unchanged. A completed provider turn, refusal, timeout, or semantic failure is not diff --git a/docs/agent-qualification-generalization-v2.md b/docs/agent-qualification-generalization-v2.md index c33a694..f9c3047 100644 --- a/docs/agent-qualification-generalization-v2.md +++ b/docs/agent-qualification-generalization-v2.md @@ -64,7 +64,8 @@ Each frozen case binds: - owner claim and excluded scope; - supported native commands; - evaluator-only gold; -- a frozen writable dependency-cache seed and manifest-declared destination; +- a frozen dependency seed and manifest-declared destination (vendored sources + for Cargo, with writable Cargo state kept separately); and - applicable evaluator adapter and machine evidence. @@ -112,7 +113,7 @@ included. Before timing, the harness must: 1. create a new origin-stripped candidate copy; -2. create a new writable copy of the case's hashed dependency seed; +2. create a new copy of the case's hashed dependency seed; 3. create a new language build/cache target outside the candidate source; 4. assert that the candidate adapter target, `.tracebook`, and declared output paths do not exist; diff --git a/experiments/agent_qualification_generalization.py b/experiments/agent_qualification_generalization.py index 1807364..1414d56 100644 --- a/experiments/agent_qualification_generalization.py +++ b/experiments/agent_qualification_generalization.py @@ -33,7 +33,7 @@ REPETITIONS = 3 AGENTS = ("codex", "claude") CONDITIONS = ("docs", "skill") -DEPENDENCY_TARGETS = ("m2-repository", "cargo-home") +DEPENDENCY_TARGETS = ("m2-repository", "cargo-home", "cargo-vendor") DOCS_TREATMENT = ( "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public " diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py index 8a0f34f..c0f5ef6 100644 --- a/experiments/agent_qualification_generalization_runs.py +++ b/experiments/agent_qualification_generalization_runs.py @@ -746,6 +746,18 @@ def _copy_fixture(case: Mapping[str, Any], scratch: Path) -> dict[str, Any]: target_name = cohort._dependency_target(declaration.get("target")) target = scratch / target_name helpers._copy_frozen_tree(source, target, str(declaration["tree_sha256"])) + if target_name == "cargo-vendor": + cargo_home = scratch / "cargo-home" + cargo_home.mkdir() + quoted_target = json.dumps(str(target)) + (cargo_home / "config.toml").write_text( + "[source.crates-io]\n" + 'replace-with = "vendored-sources"\n\n' + "[source.vendored-sources]\n" + f"directory = {quoted_target}\n\n" + "[net]\n" + "offline = true\n" + ) return { "source_manifest_sha256": declaration["manifest_sha256"], "initial_tree_sha256": declaration["tree_sha256"], diff --git a/tests/test_agent_qualification_generalization.py b/tests/test_agent_qualification_generalization.py index b5c1cb3..a253fcc 100644 --- a/tests/test_agent_qualification_generalization.py +++ b/tests/test_agent_qualification_generalization.py @@ -105,6 +105,7 @@ def test_tree_inventory_uses_same_git_exclusion_as_snapshot_digest(tmp_path): def test_dependency_target_is_explicit_and_restricted(): assert generalization._dependency_target("m2-repository") == "m2-repository" assert generalization._dependency_target("cargo-home") == "cargo-home" + assert generalization._dependency_target("cargo-vendor") == "cargo-vendor" with pytest.raises(generalization.GeneralizationError, match="dependency_target"): generalization._dependency_target("../operator-cache") diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py index ede6675..d81dc3c 100644 --- a/tests/test_agent_qualification_generalization_runs.py +++ b/tests/test_agent_qualification_generalization_runs.py @@ -85,15 +85,23 @@ def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch): "source": "private/cache", "tree_sha256": tree_hash, "manifest_sha256": "manifest", - "target": "cargo-home", + "target": "cargo-vendor", }, } fixture = execution._copy_fixture(case, scratch) assert fixture["initial_tree_sha256"] == tree_hash - assert Path(fixture["target"]).name == "cargo-home" + assert Path(fixture["target"]).name == "cargo-vendor" assert execution.helpers._snapshot_digest(Path(fixture["target"])) == tree_hash + config = scratch / "cargo-home" / "config.toml" + assert config.is_file() + assert f'directory = "{fixture["target"]}"' in config.read_text() + assert "offline = true" in config.read_text() + + (scratch / "cargo-home" / ".global-cache").write_text("mutable Cargo metadata") + execution._record_fixture_final(fixture) + assert fixture["mutated"] is False (Path(fixture["target"]) / "artifact").write_text("mutated") execution._record_fixture_final(fixture) From 02c0926f2664bde825f5211a655d5ed66b82d6f0 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Wed, 26 Aug 2026 04:57:18 -0400 Subject: [PATCH 08/19] keep qualification build state outside evidence --- docs/agent-qualification-generalization-execution-v2.md | 3 +++ experiments/agent_qualification_generalization_prompt.txt | 6 ++++++ experiments/agent_qualification_generalization_runs.py | 1 + tests/test_agent_qualification_generalization_runs.py | 1 + 4 files changed, 11 insertions(+) diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md index e6df7a7..dc588cb 100644 --- a/docs/agent-qualification-generalization-execution-v2.md +++ b/docs/agent-qualification-generalization-execution-v2.md @@ -48,6 +48,9 @@ condition-specific native skill surface. For Cargo cases, the hashed fixture is an immutable vendored-source tree while Cargo home and target directories are separate fresh writable paths. The local Tracebook checkout, private gold, prior workspaces, and prior transcripts are not mounted into the workspace. +Fresh candidate clones, cache copies, build products, tool state, and virtual +environments belong under the supplied external `FRESH_*` paths so the final +workspace evidence contains only source-sized audit artifacts and deliverables. Before accepting a Claude turn, the harness audits the provider catalog and requires no MCP servers, exactly one intended plugin, and skill visibility that diff --git a/experiments/agent_qualification_generalization_prompt.txt b/experiments/agent_qualification_generalization_prompt.txt index c0f6b44..0718fb6 100644 --- a/experiments/agent_qualification_generalization_prompt.txt +++ b/experiments/agent_qualification_generalization_prompt.txt @@ -35,6 +35,12 @@ and absent-before-use output path. Preserve both run identities and compare their deterministic results. Do not overwrite an incomplete bundle or call a second command over retained state a clean reproduction. +Keep candidate clones, dependency-store copies, build products, tool state, and +virtual environments under `FRESH_WORK_ROOT` or the supplied `FRESH_*` paths, +never inside the candidate repository. The candidate repository should retain +only source-sized audit artifacts and the requested final deliverables; do not +copy dependency stores, toolchains, or compiled outputs into those artifacts. + Deliver: 1. `CLASSIFICATION.md`: terminal class, scope, evidence, first disagreement diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py index c0f5ef6..8ad889d 100644 --- a/experiments/agent_qualification_generalization_runs.py +++ b/experiments/agent_qualification_generalization_runs.py @@ -332,6 +332,7 @@ def _fresh_environment(scratch: Path) -> dict[str, str]: "FRESH_M2": str(scratch / "m2-repository"), "FRESH_CARGO_HOME": str(scratch / "cargo-home"), "FRESH_TARGET": str(scratch / "cargo-target"), + "FRESH_WORK_ROOT": str(scratch / "fresh-work"), } diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py index d81dc3c..b568df7 100644 --- a/tests/test_agent_qualification_generalization_runs.py +++ b/tests/test_agent_qualification_generalization_runs.py @@ -308,6 +308,7 @@ def test_provider_shells_receive_fresh_paths(tmp_path, monkeypatch): assert values["FRESH_M2"] == str(tmp_path / "m2-repository") assert values["FRESH_CARGO_HOME"] == str(tmp_path / "cargo-home") assert values["FRESH_TARGET"] == str(tmp_path / "cargo-target") + assert values["FRESH_WORK_ROOT"] == str(tmp_path / "fresh-work") def test_claude_binding_probes_an_explicit_first_party_route(tmp_path, monkeypatch): From 78460591edbaf003e4d16457f3d50772b7875108 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Wed, 26 Aug 2026 06:41:53 -0400 Subject: [PATCH 09/19] isolate Maven seed from mutable local repository --- ...alification-generalization-execution-v2.md | 10 ++++--- docs/agent-qualification-generalization-v2.md | 5 ++-- .../agent_qualification_generalization.py | 2 +- ...agent_qualification_generalization_runs.py | 4 ++- ...test_agent_qualification_generalization.py | 1 + ...agent_qualification_generalization_runs.py | 28 +++++++++++++++++++ 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md index dc588cb..e6d5d63 100644 --- a/docs/agent-qualification-generalization-execution-v2.md +++ b/docs/agent-qualification-generalization-execution-v2.md @@ -44,10 +44,12 @@ for the earlier C1/C3 cohort does not apply. Every run receives a new origin-stripped workspace, new dependency-seed copy, new writable tool state and build target, isolated provider state, and -condition-specific native skill surface. For Cargo cases, the hashed fixture is -an immutable vendored-source tree while Cargo home and target directories are -separate fresh writable paths. The local Tracebook checkout, private gold, -prior workspaces, and prior transcripts are not mounted into the workspace. +condition-specific native skill surface. For Maven cases, the hashed fixture is +an immutable repository seed copied into a separate writable local repository. +For Cargo cases, the hashed fixture is an immutable vendored-source tree while +Cargo home and target directories are separate fresh writable paths. The local +Tracebook checkout, private gold, prior workspaces, and prior transcripts are +not mounted into the workspace. Fresh candidate clones, cache copies, build products, tool state, and virtual environments belong under the supplied external `FRESH_*` paths so the final workspace evidence contains only source-sized audit artifacts and deliverables. diff --git a/docs/agent-qualification-generalization-v2.md b/docs/agent-qualification-generalization-v2.md index f9c3047..3878cf8 100644 --- a/docs/agent-qualification-generalization-v2.md +++ b/docs/agent-qualification-generalization-v2.md @@ -64,8 +64,9 @@ Each frozen case binds: - owner claim and excluded scope; - supported native commands; - evaluator-only gold; -- a frozen dependency seed and manifest-declared destination (vendored sources - for Cargo, with writable Cargo state kept separately); +- a frozen dependency seed and manifest-declared destination (an immutable + Maven repository seed or vendored Cargo sources, with writable tool state + kept separately); and - applicable evaluator adapter and machine evidence. diff --git a/experiments/agent_qualification_generalization.py b/experiments/agent_qualification_generalization.py index 1414d56..0a27404 100644 --- a/experiments/agent_qualification_generalization.py +++ b/experiments/agent_qualification_generalization.py @@ -33,7 +33,7 @@ REPETITIONS = 3 AGENTS = ("codex", "claude") CONDITIONS = ("docs", "skill") -DEPENDENCY_TARGETS = ("m2-repository", "cargo-home", "cargo-vendor") +DEPENDENCY_TARGETS = ("m2-repository", "m2-seed", "cargo-home", "cargo-vendor") DOCS_TREATMENT = ( "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public " diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py index 8ad889d..6411c73 100644 --- a/experiments/agent_qualification_generalization_runs.py +++ b/experiments/agent_qualification_generalization_runs.py @@ -747,7 +747,9 @@ def _copy_fixture(case: Mapping[str, Any], scratch: Path) -> dict[str, Any]: target_name = cohort._dependency_target(declaration.get("target")) target = scratch / target_name helpers._copy_frozen_tree(source, target, str(declaration["tree_sha256"])) - if target_name == "cargo-vendor": + if target_name == "m2-seed": + shutil.copytree(target, scratch / "m2-repository") + elif target_name == "cargo-vendor": cargo_home = scratch / "cargo-home" cargo_home.mkdir() quoted_target = json.dumps(str(target)) diff --git a/tests/test_agent_qualification_generalization.py b/tests/test_agent_qualification_generalization.py index a253fcc..cbef569 100644 --- a/tests/test_agent_qualification_generalization.py +++ b/tests/test_agent_qualification_generalization.py @@ -104,6 +104,7 @@ def test_tree_inventory_uses_same_git_exclusion_as_snapshot_digest(tmp_path): def test_dependency_target_is_explicit_and_restricted(): assert generalization._dependency_target("m2-repository") == "m2-repository" + assert generalization._dependency_target("m2-seed") == "m2-seed" assert generalization._dependency_target("cargo-home") == "cargo-home" assert generalization._dependency_target("cargo-vendor") == "cargo-vendor" diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py index b568df7..5cd35b8 100644 --- a/tests/test_agent_qualification_generalization_runs.py +++ b/tests/test_agent_qualification_generalization_runs.py @@ -109,6 +109,34 @@ def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch): assert fixture["final_tree_sha256"] != fixture["initial_tree_sha256"] +def test_copy_maven_fixture_keeps_writable_repository_outside_hash(tmp_path, monkeypatch): + repository = tmp_path / "repo" + source = repository / "private" / "cache" + source.mkdir(parents=True) + (source / "artifact.jar").write_text("frozen") + tree_hash = execution.helpers._snapshot_digest(source) + scratch = tmp_path / "scratch" + scratch.mkdir() + monkeypatch.setattr(execution, "REPOSITORY_ROOT", repository) + case = { + "id": "c4-compatible", + "dependency_cache": { + "source": "private/cache", + "tree_sha256": tree_hash, + "manifest_sha256": "manifest", + "target": "m2-seed", + }, + } + + fixture = execution._copy_fixture(case, scratch) + + assert Path(fixture["target"]).name == "m2-seed" + assert (scratch / "m2-repository" / "artifact.jar").read_text() == "frozen" + (scratch / "m2-repository" / "resolver-status.properties").write_text("mutable") + execution._record_fixture_final(fixture) + assert fixture["mutated"] is False + + def test_native_surface_has_no_skill_in_docs_condition(tmp_path, monkeypatch): root = tmp_path / "run" workspace = root / "workspace" From 95576b1046e085161de861364faffa8fb163b7f2 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Wed, 26 Aug 2026 08:18:30 -0400 Subject: [PATCH 10/19] accept precreated isolated Cargo home --- experiments/agent_qualification_generalization_runs.py | 2 +- tests/test_agent_qualification_generalization_runs.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py index 6411c73..6e9c1bc 100644 --- a/experiments/agent_qualification_generalization_runs.py +++ b/experiments/agent_qualification_generalization_runs.py @@ -751,7 +751,7 @@ def _copy_fixture(case: Mapping[str, Any], scratch: Path) -> dict[str, Any]: shutil.copytree(target, scratch / "m2-repository") elif target_name == "cargo-vendor": cargo_home = scratch / "cargo-home" - cargo_home.mkdir() + cargo_home.mkdir(exist_ok=True) quoted_target = json.dumps(str(target)) (cargo_home / "config.toml").write_text( "[source.crates-io]\n" diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py index 5cd35b8..a14e4be 100644 --- a/tests/test_agent_qualification_generalization_runs.py +++ b/tests/test_agent_qualification_generalization_runs.py @@ -78,6 +78,7 @@ def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch): tree_hash = execution.helpers._snapshot_digest(source) scratch = tmp_path / "scratch" scratch.mkdir() + (scratch / "cargo-home").mkdir() monkeypatch.setattr(execution, "REPOSITORY_ROOT", repository) case = { "id": "c5-boundary", From 7e2e10f71d5e2ada9cb7a761c0571b2e57a4c5f3 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Mon, 31 Aug 2026 06:28:13 -0400 Subject: [PATCH 11/19] Add isolated L3 book replay conformance --- .../workflows/nautilus-trader-book-replay.yml | 53 +++ MANIFEST.in | 1 + README.md | 6 + docs/book-replay-conformance.md | 119 ++++++ docs/positioning.md | 8 +- examples/book_replay_adapter.py | 20 + integrations/nautilus_trader/README.md | 86 ++++ integrations/nautilus_trader/__init__.py | 1 + integrations/nautilus_trader/adapter.py | 175 ++++++++ pyproject.toml | 1 + src/tracebook/book_replay/__init__.py | 55 +++ src/tracebook/book_replay/__main__.py | 5 + src/tracebook/book_replay/cli.py | 111 +++++ src/tracebook/book_replay/compare.py | 404 ++++++++++++++++++ src/tracebook/book_replay/external.py | 306 +++++++++++++ .../book_replay/fixtures/__init__.py | 1 + .../fixtures/l3-book-replay-v1.jsonl | 17 + src/tracebook/book_replay/model.py | 379 ++++++++++++++++ src/tracebook/book_replay/protocol.py | 134 ++++++ src/tracebook/book_replay/reference.py | 199 +++++++++ tests/test_book_replay.py | 283 ++++++++++++ tests/test_external_integrations.py | 57 +++ tests/test_release_metadata.py | 1 + tools/smoke_conformance_wheel.py | 5 + 24 files changed, 2425 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/nautilus-trader-book-replay.yml create mode 100644 docs/book-replay-conformance.md create mode 100644 examples/book_replay_adapter.py create mode 100644 integrations/nautilus_trader/README.md create mode 100644 integrations/nautilus_trader/__init__.py create mode 100644 integrations/nautilus_trader/adapter.py create mode 100644 src/tracebook/book_replay/__init__.py create mode 100644 src/tracebook/book_replay/__main__.py create mode 100644 src/tracebook/book_replay/cli.py create mode 100644 src/tracebook/book_replay/compare.py create mode 100644 src/tracebook/book_replay/external.py create mode 100644 src/tracebook/book_replay/fixtures/__init__.py create mode 100644 src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl create mode 100644 src/tracebook/book_replay/model.py create mode 100644 src/tracebook/book_replay/protocol.py create mode 100644 src/tracebook/book_replay/reference.py create mode 100644 tests/test_book_replay.py diff --git a/.github/workflows/nautilus-trader-book-replay.yml b/.github/workflows/nautilus-trader-book-replay.yml new file mode 100644 index 0000000..b34e6e9 --- /dev/null +++ b/.github/workflows/nautilus-trader-book-replay.yml @@ -0,0 +1,53 @@ +name: NautilusTrader L3 book replay + +on: + workflow_dispatch: + schedule: + - cron: "43 6 * * 1" + pull_request: + paths: + - ".github/workflows/nautilus-trader-book-replay.yml" + - "integrations/nautilus_trader/**" + - "src/tracebook/book_replay/**" + - "tests/test_book_replay.py" + - "tests/test_external_integrations.py" + +permissions: + contents: read + +jobs: + pinned-v2-book: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: "3.14" + cache: pip + + - name: Install Tracebook and the pinned optional candidate + run: | + python -m pip install --upgrade pip + python -m pip install -e . "nautilus-trader==2.0.0rc3" + + - name: Verify the isolated L3 book-replay profile + run: | + python -m tracebook.book_replay run \ + src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl \ + --output /tmp/nautilus-book-replay.json \ + --timeout 10 \ + --candidate python integrations/nautilus_trader/adapter.py + python -c "import json; p=json.load(open('/tmp/nautilus-book-replay.json')); assert p['artifact_type'] == 'tracebook.book-replay.report'; assert p['conformant'] is True and p['compared_events'] == 17; assert p['candidate_engine']['version'] == '2.0.0rc3'; assert p['final_state_hash'] == '9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002'" + + - name: Run the book-replay and integration contract tests + run: python -m pytest tests/test_book_replay.py tests/test_external_integrations.py + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: nautilus-trader-book-replay + path: /tmp/nautilus-book-replay.json + if-no-files-found: error diff --git a/MANIFEST.in b/MANIFEST.in index 4d1d9a9..c7606c1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -44,4 +44,5 @@ recursive-include tests *.py recursive-include src/tracebook/visualization/web *.html *.css *.js recursive-include src/tracebook/corpus/fixtures *.json *.jsonl recursive-include src/tracebook/conformance/fixtures *.json *.jsonl +recursive-include src/tracebook/book_replay/fixtures *.jsonl include src/tracebook/py.typed diff --git a/README.md b/README.md index aa05b63..9b07bee 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,12 @@ the canonical protocol. A divergence means two configured contracts disagree. It does not, by itself, declare either project incorrect. +Order-book mirrors are tested on a deliberately different surface. The pinned +[`NautilusTrader` L3 integration](https://github.com/Taz33m/tracebook/tree/main/integrations/nautilus_trader) +passes `l3-book-replay-v1`, which exercises native deltas, queue snapshots, and +non-mutating simulated fills. That result is not matching-engine conformance; +see the [book-replay contract](https://github.com/Taz33m/tracebook/blob/main/docs/book-replay-conformance.md). + ## Failure Artifacts Every stopped campaign writes an atomic, content-addressed bundle: diff --git a/docs/book-replay-conformance.md b/docs/book-replay-conformance.md new file mode 100644 index 0000000..511ffef --- /dev/null +++ b/docs/book-replay-conformance.md @@ -0,0 +1,119 @@ +# L3 Book-Replay Conformance + +`l3-book-replay-v1` is a separate conformance surface for deterministic L3 +order-book mirrors. It exists for libraries that ingest exogenous market data +and answer depth or simulated-fill questions but do not cross submitted orders +against one another. + +It is deliberately outside `tracebook.conformance` protocol v1. A successful +book-replay report says nothing about FIFO venue matching, IOC/FOK behavior, +self-trade prevention, trade identity, or candidate order lifecycle. + +## Profile Semantics + +Every JSONL record has an `op` and `symbol`. Numbers are normalized into finite, +positive, non-exponent decimal strings. + +### Add + +```json +{"op":"add","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"2"} +``` + +The order is appended at its price level. An active ID can occur only once per +symbol; a duplicate is rejected as `DUPLICATE_ORDER_ID`. + +### Update + +```json +{"op":"update","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"4"} +``` + +An update at the existing side and price replaces quantity in place, including +an increase, and retains queue priority. A side or price change removes the old +entry and appends it at the destination level. An unknown ID is rejected as +`ORDER_NOT_ACTIVE`. + +### Delete + +```json +{"op":"delete","symbol":"TEST","order_id":1} +``` + +Deletion is keyed by active source order ID. An unknown ID is rejected as +`ORDER_NOT_ACTIVE`. + +### Probe + +```json +{"op":"probe","symbol":"TEST","side":"SELL","price":"99","quantity":"8"} +``` + +`side` is the hypothetical aggressive side. A buy walks asks from low to high; +a sell walks bids from high to low. The walk stops outside the limit price, +visits orders FIFO within a level, and returns ordered `{price, quantity}` fill +segments. A probe never mutates state. + +This is execution *simulation against mirrored liquidity*. It is not an order +submission and creates no trade IDs, positions, accounts, or order events. + +## Observable State + +After every event, an adapter returns: + +- normalized applied/rejected outcome; +- ordered simulated fills, if any; +- mirrored order count; and +- SHA-256 of the canonical full snapshot. + +Snapshots contain symbols in lexical order, bids in descending price order, +asks in ascending price order, and orders FIFO within each level. The runner +requests the full snapshot when a state hash differs so the report localizes +the first order or queue-position mismatch. + +Reports use `artifact_type = "tracebook.book-replay.report"`. The wire handshake +uses `protocol = "tracebook.book-replay"`, version `1`. Neither identity is an +alias for `tracebook.conformance`. + +## Run It + +Copy the bundled trace: + +```bash +python -m tracebook.book_replay sample /tmp/tracebook-book-replay +``` + +Run an external adapter: + +```bash +python -m tracebook.book_replay run \ + /tmp/tracebook-book-replay/l3-book-replay-v1.jsonl \ + --output /tmp/book-replay-report.json \ + --candidate-cmd './book-adapter --tracebook-book-replay-stdio' +``` + +Exit `0` means semantic agreement, `1` means the first semantic divergence was +recorded, and `2` means input, process, or protocol failure. + +Python adapters can use the server helper: + +```python +from tracebook.book_replay import serve_book_replay_stdio + +raise SystemExit(serve_book_replay_stdio(MyBookAdapter)) +``` + +The complete reference-backed example is +[`examples/book_replay_adapter.py`](../examples/book_replay_adapter.py). + +## NautilusTrader Boundary Proof + +The maintained optional adapter targets NautilusTrader's native Rust-backed +`L3_MBO` book and direct `simulate_fills(BookOrder)` Python binding. It passes +all 17 bundled events at the pinned `2.0.0rc3` release candidate. See the +[`integrations/nautilus_trader`](../integrations/nautilus_trader) proof, +provenance, exact command, and LGPL distribution warning. + +NautilusTrader's broader backtest matching engine remains out of scope here. +That engine fills candidate orders against book state supplied by market data; +it does not turn this delta profile into a submitted-order CLOB. diff --git a/docs/positioning.md b/docs/positioning.md index c19c8c8..4de461b 100644 --- a/docs/positioning.md +++ b/docs/positioning.md @@ -16,8 +16,12 @@ Strong open-source systems already own broader categories: focuses on replaying historical Level 3 orders and latency-aware HFT strategy simulation. - [NautilusTrader](https://nautilustrader.io/docs/latest/concepts/order_book/) - provides a Rust L1-L3 order book inside a much larger backtesting and live - trading platform. + provides a native Rust L1-L3 market-data book inside a much larger backtesting + and live trading platform. Its backtest matching engine simulates candidate + fills against that exogenous book; it is not a submitted-order CLOB matching + candidate for Tracebook's `fifo-*` profiles. The separate + [`l3-book-replay-v1`](book-replay-conformance.md) surface checks the book + mirror and non-mutating fill walk without making that category claim. Building a smaller general-purpose simulator beside those projects would leave Tracebook between categories: too slow and incomplete for production exchange diff --git a/examples/book_replay_adapter.py b/examples/book_replay_adapter.py new file mode 100644 index 0000000..8b288a6 --- /dev/null +++ b/examples/book_replay_adapter.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Minimal Python adapter for the isolated L3 book-replay protocol.""" + +from tracebook.book_replay import ( + EngineMetadata, + ReferenceBookReplayAdapter, + serve_book_replay_stdio, +) + + +class ExampleBookReplayAdapter(ReferenceBookReplayAdapter): + """Replace this reference-backed body with calls into a candidate L3 book.""" + + def __init__(self, config): + super().__init__(config) + self.metadata = EngineMetadata("example-book-replay-adapter", "1", "Python") + + +if __name__ == "__main__": + raise SystemExit(serve_book_replay_stdio(ExampleBookReplayAdapter)) diff --git a/integrations/nautilus_trader/README.md b/integrations/nautilus_trader/README.md new file mode 100644 index 0000000..650188b --- /dev/null +++ b/integrations/nautilus_trader/README.md @@ -0,0 +1,86 @@ +# NautilusTrader L3 Book-Replay Integration + +This optional adapter checks NautilusTrader's native Rust `OrderBook` through +Tracebook's separate `l3-book-replay-v1` surface. It does **not** run the +matching-engine protocol and does not claim that NautilusTrader is a submitted +order versus submitted order CLOB. + +The integration pins the pre-release Python binding whose public API exposes +`OrderBook.simulate_fills(BookOrder)` directly: + +- package: `nautilus-trader==2.0.0rc3`; +- tag: `v2.0.0rc3`; +- source commit: `648970ce64a304d93da0a29320cb6e19b905fa39`; +- repository: . + +The stable `1.231.0` release and the v2 release candidate are different Python +binding generations. This adapter intentionally fails closed on any installed +version other than the pin above. + +## What The Proof Exercises + +The adapter creates one native `L3_MBO` `OrderBook` per trace symbol. `add`, +`update`, and `delete` events call the corresponding native book methods. A +`probe` constructs a native `BookOrder` and calls the native +`OrderBook.simulate_fills` method. Snapshots traverse `book.bids()`, +`book.asks()`, and each level's FIFO `get_orders()` result. + +The 17-event bundled trace covers: + +- same-price FIFO insertion; +- a size increase that retains its current queue slot; +- ordered partial simulation across orders and price levels; +- a price-changing update that requeues at its destination level; +- deletion by source order ID; +- an inactive-delete rejection at the normalized adapter boundary; +- independent symbols; and +- probes that leave the native book unchanged. + +No trace order is submitted to NautilusTrader's execution engine. The only +liquidity is the exogenous L3 state built from deltas. + +## Run The Pinned Proof + +Use a separate Python 3.12-3.14 environment because the v2 candidate does not +support Tracebook's Python 3.10 floor: + +```bash +python3.14 -m venv /tmp/tracebook-nautilus +/tmp/tracebook-nautilus/bin/python -m pip install -e . \ + "nautilus-trader==2.0.0rc3" + +/tmp/tracebook-nautilus/bin/python -m tracebook.book_replay run \ + src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl \ + --output /tmp/nautilus-book-replay.json \ + --candidate /tmp/tracebook-nautilus/bin/python \ + integrations/nautilus_trader/adapter.py +``` + +The command exits `0` and the pinned candidate produces: + +```json +{ + "candidate_engine": { + "language": "Rust/Python", + "name": "NautilusTrader L3 OrderBook", + "version": "2.0.0rc3" + }, + "compared_events": 17, + "conformant": true, + "final_state_hash": "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002" +} +``` + +## License Boundary + +NautilusTrader declares `LGPL-3.0-only` in its Cargo workspace. It is not a +Tracebook dependency, no Nautilus source or binary is vendored here, and the +normal Tracebook wheel does not import it. The optional adapter loads a +user-installed Nautilus wheel in the candidate process and communicates with +the Tracebook runner over NDJSON. + +That process boundary avoids adding an LGPL Rust crate to Tracebook's build, +but it does not waive obligations for someone who redistributes a combined +bundle. In particular, a distributed binary statically linking the Nautilus +Rust crates needs an explicit LGPL section 4 compliance design. This note is a +distribution warning, not legal advice. diff --git a/integrations/nautilus_trader/__init__.py b/integrations/nautilus_trader/__init__.py new file mode 100644 index 0000000..49966f3 --- /dev/null +++ b/integrations/nautilus_trader/__init__.py @@ -0,0 +1 @@ +"""Optional NautilusTrader L3 book-replay integration.""" diff --git a/integrations/nautilus_trader/adapter.py b/integrations/nautilus_trader/adapter.py new file mode 100644 index 0000000..a903705 --- /dev/null +++ b/integrations/nautilus_trader/adapter.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""NautilusTrader v2 L3 book adapter for Tracebook's book-replay protocol.""" + +from __future__ import annotations + +from importlib import metadata as importlib_metadata + +from tracebook.book_replay import ( + BookReplayObservation, + BookReplaySnapshot, + BookReplayState, + EngineMetadata, + Outcome, + RestingBookOrder, + SimulatedFill, + serve_book_replay_stdio, +) +from tracebook.conformance import canonical_decimal + +UPSTREAM_REPOSITORY = "https://github.com/nautechsystems/nautilus_trader.git" +UPSTREAM_TAG = "v2.0.0rc3" +UPSTREAM_COMMIT = "648970ce64a304d93da0a29320cb6e19b905fa39" +UPSTREAM_VERSION = "2.0.0rc3" + + +class NautilusTraderBookReplayAdapter: + """Translate the normalized profile onto NautilusTrader's native L3 book.""" + + def __init__(self, config): + from nautilus_trader.model import ( # pylint: disable=import-outside-toplevel + BookOrder, + BookType, + InstrumentId, + OrderBook, + OrderSide, + Price, + Quantity, + ) + + installed_version = importlib_metadata.version("nautilus-trader") + if installed_version != UPSTREAM_VERSION: + raise RuntimeError( + f"NautilusTrader adapter requires {UPSTREAM_VERSION}, found {installed_version}" + ) + self.config = config + self.metadata = EngineMetadata( + "NautilusTrader L3 OrderBook", + installed_version, + "Rust/Python", + ) + self._BookOrder = BookOrder + self._BookType = BookType + self._InstrumentId = InstrumentId + self._OrderBook = OrderBook + self._OrderSide = OrderSide + self._Price = Price + self._Quantity = Quantity + self._books = {} + self._orders = {} + + def _book(self, symbol): + book = self._books.get(symbol) + if book is None: + instrument_id = self._InstrumentId.from_str(f"{symbol}.TRACEBOOK") + book = self._OrderBook(instrument_id, self._BookType.L3_MBO) + self._books[symbol] = book + self._orders[symbol] = {} + return book + + def _native_order(self, event): + side = self._OrderSide.BUY if event.side == "BUY" else self._OrderSide.SELL + return self._BookOrder( + side, + self._Price.from_str(event.price), + self._Quantity.from_str(event.quantity), + event.order_id or 0, + ) + + def apply(self, event, index): + fills: tuple[SimulatedFill, ...] = () + outcome = Outcome("applied") + known_orders = self._orders.get(event.symbol, {}) + if event.op == "add": + if event.order_id in known_orders: + outcome = Outcome( + "rejected", + "DUPLICATE_ORDER_ID", + f"Order {event.order_id} is already present in {event.symbol}", + ) + else: + book = self._book(event.symbol) + native_order = self._native_order(event) + book.add(native_order, 0, index, index) + self._orders[event.symbol][event.order_id] = native_order + elif event.op == "update": + if event.order_id not in known_orders: + outcome = Outcome( + "rejected", + "ORDER_NOT_ACTIVE", + f"Order {event.order_id} is not present in {event.symbol}", + ) + else: + book = self._books[event.symbol] + old_order = known_orders[event.order_id] + native_order = self._native_order(event) + if old_order.side != native_order.side: + # Native OrderBook.update selects one ladder from the new side. A + # normalized side move is therefore delete+add at this boundary. + book.delete(old_order, 0, index, index) + book.add(native_order, 0, index, index) + else: + book.update(native_order, 0, index, index) + known_orders[event.order_id] = native_order + elif event.op == "delete": + if event.order_id not in known_orders: + outcome = Outcome( + "rejected", + "ORDER_NOT_ACTIVE", + f"Order {event.order_id} is not present in {event.symbol}", + ) + else: + native_order = known_orders.pop(event.order_id) + self._books[event.symbol].delete(native_order, 0, index, index) + else: + book = self._books.get(event.symbol) + if book is not None: + probe = self._native_order(event) + fills = tuple( + SimulatedFill( + price=canonical_decimal(str(price)), + quantity=canonical_decimal(str(quantity)), + ) + for price, quantity in book.simulate_fills(probe) + ) + + state = self.snapshot() + return BookReplayObservation( + index=index, + outcome=outcome, + fills=fills, + state_hash=state.digest(), + order_count=state.order_count, + ) + + def snapshot(self): + snapshots = [] + for symbol in sorted(self._books): + book = self._books[symbol] + snapshots.append( + BookReplaySnapshot( + symbol=symbol, + bids=self._snapshot_levels(book.bids()), + asks=self._snapshot_levels(book.asks()), + ) + ) + return BookReplayState(tuple(snapshots)) + + @staticmethod + def _snapshot_levels(levels): + return tuple( + RestingBookOrder( + order_id=order.order_id, + price=canonical_decimal(str(order.price)), + quantity=canonical_decimal(str(order.size)), + ) + for level in levels + for order in level.get_orders() + ) + + def close(self): + """The native book objects own no external resources.""" + + +if __name__ == "__main__": + raise SystemExit(serve_book_replay_stdio(NautilusTraderBookReplayAdapter)) diff --git a/pyproject.toml b/pyproject.toml index 6d6eae8..b73e5e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ tracebook = ["py.typed"] "tracebook.corpus.fixtures" = ["coinbase-btcusd-synthetic-v1/*"] "tracebook.conformance.fixtures.v1" = ["*.json", "*.jsonl"] "tracebook.conformance.fixtures.v2" = ["*.json", "*.jsonl"] +"tracebook.book_replay.fixtures" = ["*.jsonl"] "tracebook.visualization.web" = ["*.html", "*.css", "*.js"] [tool.pytest.ini_options] diff --git a/src/tracebook/book_replay/__init__.py b/src/tracebook/book_replay/__init__.py new file mode 100644 index 0000000..0a53d52 --- /dev/null +++ b/src/tracebook/book_replay/__init__.py @@ -0,0 +1,55 @@ +"""A separate, explicitly non-CLOB L3 book-replay conformance surface.""" + +from .compare import BookReplayDivergence, BookReplayReport, run_book_replay +from .external import ( + BookReplayProtocolError, + ExternalBookReplayAdapter, + ExternalBookReplayAdapterFactory, +) +from .model import ( + ARTIFACT_SCHEMA_VERSION, + PROFILE_NAME, + PROTOCOL_NAME, + PROTOCOL_VERSION, + BookReplayConfig, + BookReplayError, + BookReplayEvent, + BookReplayObservation, + BookReplaySnapshot, + BookReplayState, + EngineMetadata, + Outcome, + RestingBookOrder, + SimulatedFill, + load_book_replay_events, +) +from .protocol import BookReplayAdapter, BookReplayAdapterFactory, serve_book_replay_stdio +from .reference import ReferenceBookReplayAdapter + +__all__ = [ + "ARTIFACT_SCHEMA_VERSION", + "PROFILE_NAME", + "PROTOCOL_NAME", + "PROTOCOL_VERSION", + "BookReplayAdapter", + "BookReplayAdapterFactory", + "BookReplayConfig", + "BookReplayDivergence", + "BookReplayError", + "BookReplayEvent", + "BookReplayObservation", + "BookReplayProtocolError", + "BookReplayReport", + "BookReplaySnapshot", + "BookReplayState", + "EngineMetadata", + "ExternalBookReplayAdapter", + "ExternalBookReplayAdapterFactory", + "Outcome", + "ReferenceBookReplayAdapter", + "RestingBookOrder", + "SimulatedFill", + "load_book_replay_events", + "run_book_replay", + "serve_book_replay_stdio", +] diff --git a/src/tracebook/book_replay/__main__.py b/src/tracebook/book_replay/__main__.py new file mode 100644 index 0000000..6a14a89 --- /dev/null +++ b/src/tracebook/book_replay/__main__.py @@ -0,0 +1,5 @@ +"""Run the book-replay CLI with ``python -m tracebook.book_replay``.""" + +from .cli import main + +raise SystemExit(main()) diff --git a/src/tracebook/book_replay/cli.py b/src/tracebook/book_replay/cli.py new file mode 100644 index 0000000..7b1446a --- /dev/null +++ b/src/tracebook/book_replay/cli.py @@ -0,0 +1,111 @@ +"""CLI for the isolated L3 book-replay conformance surface.""" + +from __future__ import annotations + +import argparse +import json +import shlex +import sys +from importlib import resources +from pathlib import Path +from typing import List, Optional + +from .._version import __version__ +from .compare import run_book_replay +from .external import ExternalBookReplayAdapterFactory +from .model import PROFILE_NAME, BookReplayConfig, BookReplayError, load_book_replay_events + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m tracebook.book_replay", + description=( + "Compare an external L3 order-book mirror without claiming matching-engine " + "conformance." + ), + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + commands = parser.add_subparsers(dest="command", required=True) + + sample = commands.add_parser("sample", help="Copy the bundled L3 replay trace.") + sample.add_argument("destination", help="New directory for the profile trace.") + + run = commands.add_parser("run", help="Compare one L3 delta/probe JSONL trace.") + run.add_argument("events") + run.add_argument("--output") + run.add_argument("--profile", choices=[PROFILE_NAME], default=PROFILE_NAME) + run.add_argument("--timeout", type=float, default=5.0) + candidate = run.add_mutually_exclusive_group(required=True) + candidate.add_argument("--candidate-cmd", help="Candidate command as one shell-style string.") + candidate.add_argument( + "--candidate", + nargs=argparse.REMAINDER, + help="Adapter command and arguments; this must be the final CLI option.", + ) + return parser + + +def _candidate_factory(args) -> ExternalBookReplayAdapterFactory: + command = ( + shlex.split(args.candidate_cmd) if args.candidate_cmd is not None else list(args.candidate) + ) + if command and command[0] == "--": + command = command[1:] + if not command: + raise BookReplayError("--candidate requires a command") + return ExternalBookReplayAdapterFactory(command, timeout_seconds=args.timeout) + + +def _copy_sample(destination: str) -> Path: + root = Path(destination) + target = root / f"{PROFILE_NAME}.jsonl" + if root.exists() and not root.is_dir(): + raise BookReplayError("sample destination must be a directory") + if target.exists(): + raise BookReplayError(f"sample trace already exists: {target}") + root.mkdir(parents=True, exist_ok=True) + source = resources.files("tracebook.book_replay.fixtures").joinpath(f"{PROFILE_NAME}.jsonl") + target.write_bytes(source.read_bytes()) + return target + + +def _emit(payload: dict, output: Optional[str]) -> None: + rendered = json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n" + if output is None: + print(rendered, end="") + return + path = Path(output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered, encoding="utf-8") + print(f"Report written: {path}") + + +def main(argv: Optional[List[str]] = None) -> int: + args = _build_parser().parse_args(argv) + try: + if args.command == "sample": + target = _copy_sample(args.destination) + print(f"Book-replay trace copied to: {target}") + print(f"Profile: {PROFILE_NAME}") + return 0 + + events_path = Path(args.events).expanduser().resolve() + if args.output is not None and Path(args.output).expanduser().resolve() == events_path: + raise BookReplayError("input and output paths must be distinct") + report = run_book_replay( + load_book_replay_events(str(events_path)), + _candidate_factory(args), + config=BookReplayConfig(args.profile), + trace_name=str(events_path), + ) + payload = report.to_dict() + _emit(payload, args.output) + if report.operational_failure: + return 2 + return 0 if report.conformant else 1 + except (BookReplayError, OSError, TypeError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 2 + + +__all__ = ["main"] diff --git a/src/tracebook/book_replay/compare.py b/src/tracebook/book_replay/compare.py new file mode 100644 index 0000000..c64e726 --- /dev/null +++ b/src/tracebook/book_replay/compare.py @@ -0,0 +1,404 @@ +"""Compare one L3 replay trace and localize the first observable drift.""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass, replace +from typing import Any, Iterable, Optional + +from ..conformance.model import first_difference +from .model import ( + ARTIFACT_SCHEMA_VERSION, + PROTOCOL_VERSION, + BookReplayConfig, + BookReplayError, + BookReplayEvent, + BookReplayObservation, + BookReplayState, + EngineMetadata, + trace_sha256, +) +from .protocol import BookReplayAdapter, BookReplayAdapterFactory +from .reference import ReferenceBookReplayAdapter + + +@dataclass(frozen=True) +class BookReplayDivergence: + """First observable disagreement in a book-replay trace.""" + + event_index: int + category: str + kind: str + path: str + message: str + event: Optional[dict] + reference: Any + candidate: Any + snapshot_error: Optional[str] = None + close_error: Optional[str] = None + + def to_dict(self) -> dict: + payload = { + "event_index": self.event_index, + "category": self.category, + "kind": self.kind, + "path": self.path, + "message": self.message, + "event": self.event, + "reference": self.reference, + "candidate": self.candidate, + "snapshot_error": self.snapshot_error, + } + if self.close_error is not None: + payload["close_error"] = self.close_error + return payload + + +@dataclass(frozen=True) +class BookReplayReport: + """Stable JSON artifact for one isolated L3 replay comparison.""" + + config: BookReplayConfig + trace_name: Optional[str] + trace_hash: str + event_count: int + compared_events: int + reference_engine: EngineMetadata + candidate_engine: EngineMetadata + conformant: bool + final_state_hash: Optional[str] + divergence: Optional[BookReplayDivergence] + + @property + def operational_failure(self) -> bool: + return self.divergence is not None and ( + self.divergence.category == "protocol" + or self.divergence.snapshot_error is not None + or self.divergence.close_error is not None + ) + + def to_dict(self) -> dict: + return { + "schema_version": ARTIFACT_SCHEMA_VERSION, + "artifact_type": "tracebook.book-replay.report", + "protocol_version": PROTOCOL_VERSION, + "trace": { + "name": self.trace_name, + "sha256": self.trace_hash, + "event_count": self.event_count, + }, + "config": self.config.to_dict(), + "reference_engine": self.reference_engine.to_dict(), + "candidate_engine": self.candidate_engine.to_dict(), + "compared_events": self.compared_events, + "conformant": self.conformant, + "final_state_hash": self.final_state_hash, + "divergence": self.divergence.to_dict() if self.divergence else None, + } + + +def _divergence( + event: BookReplayEvent, + index: int, + category: str, + difference: dict, + message: str, +) -> BookReplayDivergence: + return BookReplayDivergence( + event_index=index, + category=category, + kind=difference["kind"], + path=difference["path"], + message=message, + event=event.to_dict(), + reference=difference.get("reference"), + candidate=difference.get("candidate"), + ) + + +def _compare_observations( + event: BookReplayEvent, + index: int, + reference: BookReplayObservation, + candidate: BookReplayObservation, +) -> Optional[BookReplayDivergence]: + reference_outcome = { + "status": reference.outcome.status, + "reason": reference.outcome.reason, + } + candidate_outcome = { + "status": candidate.outcome.status, + "reason": candidate.outcome.reason, + } + difference = first_difference(reference_outcome, candidate_outcome, "$.observation.outcome") + if difference is not None: + return _divergence( + event, + index, + "outcome", + difference, + "candidate delta acceptance differs from the reference", + ) + + difference = first_difference( + [fill.to_dict() for fill in reference.fills], + [fill.to_dict() for fill in candidate.fills], + "$.observation.fills", + ) + if difference is not None: + return _divergence( + event, + index, + "simulated_fills", + difference, + "candidate simulated fills differ from the reference", + ) + + if reference.order_count != candidate.order_count: + return BookReplayDivergence( + event_index=index, + category="book_state", + kind="value_mismatch", + path="$.observation.order_count", + message="candidate mirrored-order count differs from the reference", + event=event.to_dict(), + reference=reference.order_count, + candidate=candidate.order_count, + ) + if reference.state_hash != candidate.state_hash: + return BookReplayDivergence( + event_index=index, + category="book_state", + kind="value_mismatch", + path="$.observation.state_hash", + message="candidate mirrored state differs from the reference", + event=event.to_dict(), + reference=reference.state_hash, + candidate=candidate.state_hash, + ) + return None + + +def _localize_state( + divergence: BookReplayDivergence, + reference: ReferenceBookReplayAdapter, + candidate: BookReplayAdapter, + candidate_observation: Optional[BookReplayObservation], +) -> BookReplayDivergence: + try: + reference_state = reference.snapshot() + candidate_state = candidate.snapshot() + if not isinstance(candidate_state, BookReplayState): + raise BookReplayError("candidate snapshot must return BookReplayState") + except Exception as exc: + return BookReplayDivergence( + event_index=divergence.event_index, + category="protocol", + kind="adapter_error", + path="$.state", + message=f"candidate snapshot failed: {exc}", + event=divergence.event, + reference="valid candidate snapshot", + candidate=None, + snapshot_error=f"candidate snapshot failed: {exc}", + ) + + candidate_digest = candidate_state.digest() + if candidate_observation is not None and candidate_digest != candidate_observation.state_hash: + return BookReplayDivergence( + event_index=divergence.event_index, + category="protocol", + kind="invalid_state_hash", + path="$.observation.state_hash", + message="candidate state_hash does not describe its snapshot", + event=divergence.event, + reference=candidate_digest, + candidate=candidate_observation.state_hash, + ) + difference = first_difference(reference_state.to_dict(), candidate_state.to_dict(), "$.state") + if difference is None: + return divergence + return BookReplayDivergence( + event_index=divergence.event_index, + category="book_state", + kind=difference["kind"], + path=difference["path"], + message="candidate L3 orders or queue priority differ from the reference", + event=divergence.event, + reference=difference.get("reference"), + candidate=difference.get("candidate"), + ) + + +def _protocol_divergence( + event: BookReplayEvent, + index: int, + exc: Exception, +) -> BookReplayDivergence: + return BookReplayDivergence( + event_index=index, + category="protocol", + kind="adapter_error", + path="$", + message=str(exc), + event=event.to_dict(), + reference="valid observation", + candidate=None, + ) + + +def run_book_replay( + events: Iterable[BookReplayEvent], + candidate_factory: BookReplayAdapterFactory, + config: Optional[BookReplayConfig] = None, + trace_name: Optional[str] = None, +) -> BookReplayReport: + """Compare one candidate book mirror and stop at its first drift.""" + normalized_events = list(events) + if any(not isinstance(event, BookReplayEvent) for event in normalized_events): + raise BookReplayError("book-replay traces contain only BookReplayEvent values") + config = config or BookReplayConfig() + reference = ReferenceBookReplayAdapter(config) + candidate = candidate_factory(config) + candidate_metadata = getattr(candidate, "metadata", None) + if not isinstance(candidate_metadata, EngineMetadata): + with suppress(Exception): + candidate.close() + reference.close() + raise BookReplayError("candidate adapter metadata must be EngineMetadata") + + divergence: Optional[BookReplayDivergence] = None + compared_events = 0 + final_state_hash: Optional[str] = None + last_candidate_observation: Optional[BookReplayObservation] = None + close_error: Optional[Exception] = None + + try: + for index, event in enumerate(normalized_events, 1): + reference_observation = reference.apply(event, index) + try: + candidate_observation = candidate.apply(event, index) + if not isinstance(candidate_observation, BookReplayObservation): + raise BookReplayError("candidate apply() must return BookReplayObservation") + if candidate_observation.index != index: + raise BookReplayError( + f"candidate observation index {candidate_observation.index} " + f"does not match event {index}" + ) + except Exception as exc: + divergence = _protocol_divergence(event, index, exc) + compared_events = index + break + last_candidate_observation = candidate_observation + compared_events = index + divergence = _compare_observations( + event, + index, + reference_observation, + candidate_observation, + ) + if divergence is not None: + if divergence.category == "book_state": + divergence = _localize_state( + divergence, + reference, + candidate, + candidate_observation, + ) + break + + if divergence is None: + reference_state = reference.snapshot() + final_state_hash = reference_state.digest() + try: + candidate_state = candidate.snapshot() + if not isinstance(candidate_state, BookReplayState): + raise BookReplayError("candidate snapshot must return BookReplayState") + candidate_digest = candidate_state.digest() + if ( + last_candidate_observation is not None + and candidate_digest != last_candidate_observation.state_hash + ): + hashed_event = normalized_events[-1] + divergence = BookReplayDivergence( + event_index=len(normalized_events), + category="protocol", + kind="invalid_state_hash", + path="$.observation.state_hash", + message="candidate final state_hash does not describe its snapshot", + event=hashed_event.to_dict(), + reference=candidate_digest, + candidate=last_candidate_observation.state_hash, + ) + else: + difference = first_difference( + reference_state.to_dict(), + candidate_state.to_dict(), + "$.state", + ) + if difference is not None: + state_event = normalized_events[-1] if normalized_events else None + divergence = BookReplayDivergence( + event_index=len(normalized_events), + category="book_state", + kind=difference["kind"], + path=difference["path"], + message="candidate final L3 state differs from the reference", + event=state_event.to_dict() if state_event else None, + reference=difference.get("reference"), + candidate=difference.get("candidate"), + ) + except Exception as exc: + failed_event = normalized_events[-1] if normalized_events else None + divergence = BookReplayDivergence( + event_index=len(normalized_events), + category="protocol", + kind="adapter_error", + path="$.state", + message=f"candidate final snapshot failed: {exc}", + event=failed_event.to_dict() if failed_event else None, + reference="valid final snapshot", + candidate=None, + ) + finally: + try: + candidate.close() + except Exception as exc: + close_error = exc + reference.close() + + if close_error is not None: + if divergence is None: + closed_event = normalized_events[compared_events - 1] if compared_events else None + divergence = BookReplayDivergence( + event_index=compared_events, + category="protocol", + kind="adapter_close_error", + path="$", + message=f"candidate close failed: {close_error}", + event=closed_event.to_dict() if closed_event else None, + reference="clean adapter shutdown", + candidate=None, + ) + else: + divergence = replace(divergence, close_error=f"candidate close failed: {close_error}") + + return BookReplayReport( + config=config, + trace_name=trace_name, + trace_hash=trace_sha256(normalized_events), + event_count=len(normalized_events), + compared_events=compared_events, + reference_engine=reference.metadata, + candidate_engine=candidate_metadata, + conformant=divergence is None, + final_state_hash=final_state_hash, + divergence=divergence, + ) + + +__all__ = [ + "BookReplayDivergence", + "BookReplayReport", + "run_book_replay", +] diff --git a/src/tracebook/book_replay/external.py b/src/tracebook/book_replay/external.py new file mode 100644 index 0000000..567922d --- /dev/null +++ b/src/tracebook/book_replay/external.py @@ -0,0 +1,306 @@ +"""External-process client for the isolated book-replay protocol.""" + +from __future__ import annotations + +import json +import math +import queue +import subprocess # nosec B404 +import threading +from collections import deque +from pathlib import Path +from typing import Deque, Mapping, Optional, Sequence + +from .model import ( + PROTOCOL_NAME, + PROTOCOL_VERSION, + BookReplayConfig, + BookReplayError, + BookReplayEvent, + BookReplayObservation, + BookReplayState, + EngineMetadata, +) + +_END_OF_STREAM = object() + + +class BookReplayProtocolError(BookReplayError): + """Raised when an external book adapter violates or times out on protocol.""" + + +class ExternalBookReplayAdapter: + """Drive one external book mirror over newline-delimited JSON.""" + + def __init__( + self, + command: Sequence[str], + config: BookReplayConfig, + timeout_seconds: float = 5.0, + cwd: Optional[str | Path] = None, + ) -> None: + if isinstance(command, (str, bytes)) or not command: + raise BookReplayError("candidate command must be a non-empty argument list") + normalized_command = [] + for argument in command: + if not isinstance(argument, str) or not argument: + raise BookReplayError("candidate command arguments must be non-empty strings") + normalized_command.append(argument) + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(float(timeout_seconds)) + or timeout_seconds <= 0 + ): + raise BookReplayError("timeout_seconds must be a positive finite number") + + self.command = tuple(normalized_command) + self.timeout_seconds = float(timeout_seconds) + self._closed = False + self._broken = False + self._last_index = 0 + self._stdout_queue: queue.Queue[object] = queue.Queue(maxsize=64) + self._stderr_lines: Deque[str] = deque(maxlen=64) + self._stderr_lock = threading.Lock() + try: + self._process = subprocess.Popen( # nosec B603 + self.command, + cwd=str(cwd) if cwd is not None else None, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + bufsize=1, + ) + except OSError as exc: + raise BookReplayProtocolError( + f"unable to start candidate command {self.command!r}: {exc}" + ) from exc + stdout = self._process.stdout + stderr = self._process.stderr + if stdout is None or stderr is None: + self._process.kill() + self._process.wait() + raise BookReplayProtocolError("candidate process pipes were not created") + self._stdout_thread = threading.Thread( + target=self._read_stdout, + args=(stdout,), + name="tracebook-book-replay-stdout", + daemon=True, + ) + self._stderr_thread = threading.Thread( + target=self._read_stderr, + args=(stderr,), + name="tracebook-book-replay-stderr", + daemon=True, + ) + self._stdout_thread.start() + self._stderr_thread.start() + + try: + self._send( + { + "type": "hello", + "protocol": PROTOCOL_NAME, + "protocol_version": PROTOCOL_VERSION, + "config": config.to_dict(), + } + ) + ready = self._receive("ready") + if ready.get("protocol") != PROTOCOL_NAME: + raise BookReplayProtocolError("ready message reported the wrong protocol") + if ready.get("protocol_version") != PROTOCOL_VERSION: + raise BookReplayProtocolError("ready message reported the wrong protocol version") + self.metadata = EngineMetadata.from_dict(ready.get("engine", {})) + except Exception: + self._broken = True + self._shutdown() + raise + + def _read_stdout(self, stream) -> None: + try: + for line in stream: + self._stdout_queue.put(line) + finally: + self._stdout_queue.put(_END_OF_STREAM) + + def _read_stderr(self, stream) -> None: + for line in stream: + with self._stderr_lock: + self._stderr_lines.append(line[-4096:]) + + def _stderr_tail(self) -> str: + with self._stderr_lock: + rendered = "".join(self._stderr_lines).strip() + return rendered[-4000:] if rendered else "" + + def _send(self, payload: Mapping[str, object]) -> None: + if self._closed or self._process.poll() is not None: + raise BookReplayProtocolError( + f"candidate process is not running{self._process_context()}" + ) + process_input = self._process.stdin + if process_input is None: + self._broken = True + raise BookReplayProtocolError("candidate process input pipe is unavailable") + try: + process_input.write(json.dumps(payload, separators=(",", ":"), allow_nan=False) + "\n") + process_input.flush() + except (BrokenPipeError, OSError) as exc: + self._broken = True + raise BookReplayProtocolError( + f"candidate process closed its input{self._process_context()}" + ) from exc + + def _receive(self, expected_type: str) -> dict: + try: + item = self._stdout_queue.get(timeout=self.timeout_seconds) + except queue.Empty as exc: + self._broken = True + raise BookReplayProtocolError( + f"candidate timed out waiting for {expected_type!r} after " + f"{self.timeout_seconds:g}s{self._process_context()}" + ) from exc + if item is _END_OF_STREAM: + self._broken = True + raise BookReplayProtocolError( + f"candidate exited before {expected_type!r}{self._process_context()}" + ) + if not isinstance(item, str): + self._broken = True + raise BookReplayProtocolError("candidate produced an invalid protocol frame") + try: + message = json.loads(item) + except json.JSONDecodeError as exc: + self._broken = True + raise BookReplayProtocolError( + f"candidate stdout was not valid JSON: {item.strip()!r}" + ) from exc + if not isinstance(message, dict): + self._broken = True + raise BookReplayProtocolError("candidate protocol frame must be a JSON object") + if message.get("type") == "error": + self._broken = True + raise BookReplayProtocolError( + f"candidate reported {message.get('code', 'ERROR')}: " + f"{message.get('message', '')}" + ) + if message.get("type") != expected_type: + self._broken = True + raise BookReplayProtocolError( + f"expected candidate message {expected_type!r}, " + f"received {message.get('type')!r}" + ) + return message + + def _process_context(self) -> str: + return_code = self._process.poll() + details = f" (exit code {return_code})" if return_code is not None else "" + stderr = self._stderr_tail() + return details + (f"; stderr: {stderr}" if stderr else "") + + def apply(self, event: BookReplayEvent, index: int) -> BookReplayObservation: + self._send({"type": "event", "index": index, "event": event.to_dict()}) + observation = BookReplayObservation.from_dict(self._receive("observation")) + if observation.index != index: + self._broken = True + raise BookReplayProtocolError( + f"candidate observation index {observation.index} does not match {index}" + ) + self._last_index = index + return observation + + def snapshot(self) -> BookReplayState: + self._send({"type": "snapshot", "index": self._last_index}) + message = self._receive("snapshot") + if message.get("index") != self._last_index: + self._broken = True + raise BookReplayProtocolError("candidate snapshot reported the wrong event index") + return BookReplayState.from_dict(message.get("state", {})) + + def close(self) -> None: + if self._closed: + return + close_error: Optional[BookReplayProtocolError] = None + was_broken = self._broken + try: + if not self._broken and self._process.poll() is None: + self._send({"type": "finish", "event_count": self._last_index}) + complete = self._receive("complete") + if complete.get("event_count") != self._last_index: + raise BookReplayProtocolError( + "candidate complete message reported the wrong event_count" + ) + elif not self._broken: + raise BookReplayProtocolError("candidate exited before finish") + except (BookReplayProtocolError, OSError) as exc: + self._broken = True + close_error = ( + exc + if isinstance(exc, BookReplayProtocolError) + else BookReplayProtocolError(f"candidate shutdown failed: {exc}") + ) + finally: + self._closed = True + self._shutdown() + if not was_broken and close_error is None and self._process.returncode != 0: + close_error = BookReplayProtocolError( + f"candidate exited with code {self._process.returncode} after complete" + ) + if close_error is not None: + raise close_error + + def _shutdown(self) -> None: + if self._process.poll() is None: + try: + self._process.wait(timeout=0.5) + except subprocess.TimeoutExpired: + self._process.terminate() + try: + self._process.wait(timeout=0.5) + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait(timeout=0.5) + if self._process.stdin is not None: + try: + self._process.stdin.close() + except OSError: + pass + self._stdout_thread.join(timeout=0.2) + self._stderr_thread.join(timeout=0.2) + for stream in (self._process.stdout, self._process.stderr): + if stream is not None: + try: + stream.close() + except OSError: + pass + + +class ExternalBookReplayAdapterFactory: + """Start a fresh external book adapter for each comparison.""" + + def __init__( + self, + command: Sequence[str], + timeout_seconds: float = 5.0, + cwd: Optional[str | Path] = None, + ) -> None: + self.command = tuple(command) + self.timeout_seconds = timeout_seconds + self.cwd = cwd + + def __call__(self, config: BookReplayConfig) -> ExternalBookReplayAdapter: + return ExternalBookReplayAdapter( + self.command, + config, + timeout_seconds=self.timeout_seconds, + cwd=self.cwd, + ) + + +__all__ = [ + "BookReplayProtocolError", + "ExternalBookReplayAdapter", + "ExternalBookReplayAdapterFactory", +] diff --git a/src/tracebook/book_replay/fixtures/__init__.py b/src/tracebook/book_replay/fixtures/__init__.py new file mode 100644 index 0000000..f752d3c --- /dev/null +++ b/src/tracebook/book_replay/fixtures/__init__.py @@ -0,0 +1 @@ +"""Bundled deterministic traces for the book-replay profile.""" diff --git a/src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl b/src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl new file mode 100644 index 0000000..0d99d2a --- /dev/null +++ b/src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl @@ -0,0 +1,17 @@ +{"op":"add","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"2"} +{"op":"add","symbol":"TEST","order_id":2,"side":"BUY","price":"100","quantity":"3"} +{"op":"update","symbol":"TEST","order_id":1,"side":"BUY","price":"100","quantity":"4"} +{"op":"add","symbol":"TEST","order_id":3,"side":"BUY","price":"99","quantity":"5"} +{"op":"add","symbol":"TEST","order_id":10,"side":"SELL","price":"101","quantity":"1.5"} +{"op":"add","symbol":"TEST","order_id":11,"side":"SELL","price":"101","quantity":"2.5"} +{"op":"add","symbol":"TEST","order_id":12,"side":"SELL","price":"102","quantity":"4"} +{"op":"probe","symbol":"TEST","side":"BUY","price":"102","quantity":"5"} +{"op":"probe","symbol":"TEST","side":"SELL","price":"99","quantity":"8"} +{"op":"update","symbol":"TEST","order_id":2,"side":"BUY","price":"99","quantity":"3"} +{"op":"probe","symbol":"TEST","side":"SELL","price":"99","quantity":"12"} +{"op":"delete","symbol":"TEST","order_id":1} +{"op":"delete","symbol":"TEST","order_id":10} +{"op":"probe","symbol":"TEST","side":"BUY","price":"101","quantity":"3"} +{"op":"add","symbol":"ALT","order_id":1,"side":"SELL","price":"50","quantity":"7"} +{"op":"probe","symbol":"ALT","side":"BUY","price":"50","quantity":"2"} +{"op":"delete","symbol":"TEST","order_id":999} diff --git a/src/tracebook/book_replay/model.py b/src/tracebook/book_replay/model.py new file mode 100644 index 0000000..96c0ef9 --- /dev/null +++ b/src/tracebook/book_replay/model.py @@ -0,0 +1,379 @@ +"""Versioned contracts for deterministic L3 order-book replay checks.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from decimal import Decimal +from numbers import Integral +from typing import Any, Iterable, Mapping, Optional, Tuple, cast + +from ..conformance.model import EngineMetadata, Outcome, canonical_decimal + +PROTOCOL_NAME = "tracebook.book-replay" +PROTOCOL_VERSION = 1 +ARTIFACT_SCHEMA_VERSION = 1 +PROFILE_NAME = "l3-book-replay-v1" +_HASH_LENGTH = 64 + + +class BookReplayError(ValueError): + """Raised when a book-replay trace, contract, or adapter is invalid.""" + + +def _positive_int(value: Any, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, Integral) or value <= 0: + raise BookReplayError(f"{field_name} must be a positive integer") + return int(value) + + +def _nonnegative_int(value: Any, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, Integral) or value < 0: + raise BookReplayError(f"{field_name} must be a non-negative integer") + return int(value) + + +def _positive_decimal(value: Any, field_name: str) -> str: + try: + normalized = canonical_decimal(value) + except ValueError as exc: + raise BookReplayError(str(exc)) from exc + if Decimal(normalized) <= 0: + raise BookReplayError(f"{field_name} must be positive") + return normalized + + +def _symbol(value: Any) -> str: + if not isinstance(value, str) or not value.strip(): + raise BookReplayError("symbol must be a non-empty string") + return value.strip() + + +def _side(value: Any) -> str: + if not isinstance(value, str): + raise BookReplayError("side must be BUY or SELL") + normalized = value.strip().upper() + if normalized not in {"BUY", "SELL"}: + raise BookReplayError("side must be BUY or SELL") + return normalized + + +@dataclass(frozen=True) +class BookReplayConfig: + """The immutable semantic profile selected for one comparison.""" + + profile: str = PROFILE_NAME + + def __post_init__(self) -> None: + if self.profile != PROFILE_NAME: + raise BookReplayError(f"profile must be {PROFILE_NAME!r}") + + def to_dict(self) -> dict: + return {"profile": self.profile} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BookReplayConfig": + if not isinstance(data, Mapping): + raise BookReplayError("config must be an object") + return cls(profile=data.get("profile", PROFILE_NAME)) + + +@dataclass(frozen=True) +class BookReplayEvent: + """One L3 delta or non-mutating simulated-fill probe.""" + + op: str + symbol: str + order_id: Optional[int] = None + side: Optional[str] = None + price: Optional[str] = None + quantity: Optional[str] = None + + def __post_init__(self) -> None: + if not isinstance(self.op, str): + raise BookReplayError("op must be add, update, delete, or probe") + op = self.op.strip().lower() + if op not in {"add", "update", "delete", "probe"}: + raise BookReplayError("op must be add, update, delete, or probe") + object.__setattr__(self, "op", op) + object.__setattr__(self, "symbol", _symbol(self.symbol)) + + if op in {"add", "update", "delete"}: + object.__setattr__(self, "order_id", _positive_int(self.order_id, "order_id")) + elif self.order_id is not None: + raise BookReplayError("probe events cannot contain order_id") + + if op in {"add", "update", "probe"}: + object.__setattr__(self, "side", _side(self.side)) + object.__setattr__(self, "price", _positive_decimal(self.price, "price")) + object.__setattr__( + self, + "quantity", + _positive_decimal(self.quantity, "quantity"), + ) + elif any(value is not None for value in (self.side, self.price, self.quantity)): + raise BookReplayError("delete events contain only op, symbol, and order_id") + + def to_dict(self) -> dict: + payload: dict[str, Any] = {"op": self.op, "symbol": self.symbol} + if self.order_id is not None: + payload["order_id"] = self.order_id + if self.side is not None: + payload["side"] = self.side + if self.price is not None: + payload["price"] = self.price + if self.quantity is not None: + payload["quantity"] = self.quantity + return payload + + @classmethod + def from_mapping(cls, data: Mapping[str, Any]) -> "BookReplayEvent": + if not isinstance(data, Mapping): + raise BookReplayError("book-replay event must be an object") + op = data.get("op", "") + normalized_op = op.strip().lower() if isinstance(op, str) else op + allowed = { + "add": {"op", "symbol", "order_id", "side", "price", "quantity"}, + "update": {"op", "symbol", "order_id", "side", "price", "quantity"}, + "delete": {"op", "symbol", "order_id"}, + "probe": {"op", "symbol", "side", "price", "quantity"}, + }.get(normalized_op) + if allowed is not None: + unknown = set(data) - allowed + if unknown: + raise BookReplayError( + f"{normalized_op} event contains unsupported fields: {sorted(unknown)!r}" + ) + return cls( + op=cast(str, op), + symbol=cast(str, data.get("symbol", "")), + order_id=cast(Optional[int], data.get("order_id")), + side=cast(Optional[str], data.get("side")), + price=cast(Optional[str], data.get("price")), + quantity=cast(Optional[str], data.get("quantity")), + ) + + +@dataclass(frozen=True) +class SimulatedFill: + """One price/quantity segment returned by a non-mutating probe.""" + + price: str + quantity: str + + def __post_init__(self) -> None: + object.__setattr__(self, "price", _positive_decimal(self.price, "fill price")) + object.__setattr__(self, "quantity", _positive_decimal(self.quantity, "fill quantity")) + + def to_dict(self) -> dict: + return {"price": self.price, "quantity": self.quantity} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "SimulatedFill": + if not isinstance(data, Mapping): + raise BookReplayError("simulated fill must be an object") + return cls(price=cast(str, data.get("price")), quantity=cast(str, data.get("quantity"))) + + +@dataclass(frozen=True) +class RestingBookOrder: + """One mirrored L3 order in queue position.""" + + order_id: int + price: str + quantity: str + + def __post_init__(self) -> None: + object.__setattr__(self, "order_id", _positive_int(self.order_id, "order_id")) + object.__setattr__(self, "price", _positive_decimal(self.price, "order price")) + object.__setattr__(self, "quantity", _positive_decimal(self.quantity, "order quantity")) + + def to_dict(self) -> dict: + return {"order_id": self.order_id, "price": self.price, "quantity": self.quantity} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "RestingBookOrder": + if not isinstance(data, Mapping): + raise BookReplayError("resting order must be an object") + return cls( + order_id=cast(int, data.get("order_id")), + price=cast(str, data.get("price")), + quantity=cast(str, data.get("quantity")), + ) + + +@dataclass(frozen=True) +class BookReplaySnapshot: + """Both price-time queues for one mirrored symbol.""" + + symbol: str + bids: Tuple[RestingBookOrder, ...] = () + asks: Tuple[RestingBookOrder, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "symbol", _symbol(self.symbol)) + order_ids = [order.order_id for order in self.bids + self.asks] + if len(order_ids) != len(set(order_ids)): + raise BookReplayError(f"book {self.symbol!r} contains duplicate order ids") + + def to_dict(self) -> dict: + return { + "symbol": self.symbol, + "bids": [order.to_dict() for order in self.bids], + "asks": [order.to_dict() for order in self.asks], + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BookReplaySnapshot": + if not isinstance(data, Mapping): + raise BookReplayError("book snapshot must be an object") + bids = data.get("bids", []) + asks = data.get("asks", []) + if not isinstance(bids, list) or not isinstance(asks, list): + raise BookReplayError("book bids and asks must be arrays") + return cls( + symbol=cast(str, data.get("symbol", "")), + bids=tuple(RestingBookOrder.from_dict(item) for item in bids), + asks=tuple(RestingBookOrder.from_dict(item) for item in asks), + ) + + +@dataclass(frozen=True) +class BookReplayState: + """Canonical mirrored state, including L3 queue order.""" + + books: Tuple[BookReplaySnapshot, ...] = () + + def __post_init__(self) -> None: + symbols = [book.symbol for book in self.books] + if symbols != sorted(symbols) or len(symbols) != len(set(symbols)): + raise BookReplayError("book snapshots must have unique symbols in sorted order") + + @property + def order_count(self) -> int: + return sum(len(book.bids) + len(book.asks) for book in self.books) + + def to_dict(self) -> dict: + return {"books": [book.to_dict() for book in self.books]} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BookReplayState": + if not isinstance(data, Mapping): + raise BookReplayError("state must be an object") + books = data.get("books", []) + if not isinstance(books, list): + raise BookReplayError("state books must be an array") + return cls(tuple(BookReplaySnapshot.from_dict(item) for item in books)) + + def digest(self) -> str: + return hashlib.sha256(_canonical_json(self.to_dict())).hexdigest() + + +@dataclass(frozen=True) +class BookReplayObservation: + """Comparable result emitted after one delta or probe.""" + + index: int + outcome: Outcome + fills: Tuple[SimulatedFill, ...] + state_hash: str + order_count: int + + def __post_init__(self) -> None: + object.__setattr__(self, "index", _positive_int(self.index, "observation index")) + if ( + not isinstance(self.state_hash, str) + or len(self.state_hash) != _HASH_LENGTH + or any(character not in "0123456789abcdef" for character in self.state_hash) + ): + raise BookReplayError("state_hash must be a lowercase SHA-256 hex digest") + object.__setattr__( + self, + "order_count", + _nonnegative_int(self.order_count, "order_count"), + ) + + def to_dict(self, include_type: bool = False) -> dict: + payload = { + "index": self.index, + "outcome": self.outcome.to_dict(), + "fills": [fill.to_dict() for fill in self.fills], + "state_hash": self.state_hash, + "order_count": self.order_count, + } + if include_type: + payload = {"type": "observation", **payload} + return payload + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "BookReplayObservation": + if not isinstance(data, Mapping): + raise BookReplayError("observation must be an object") + fills = data.get("fills", []) + if not isinstance(fills, list): + raise BookReplayError("observation fills must be an array") + return cls( + index=cast(int, data.get("index")), + outcome=Outcome.from_dict(data.get("outcome", {})), + fills=tuple(SimulatedFill.from_dict(item) for item in fills), + state_hash=cast(str, data.get("state_hash")), + order_count=cast(int, data.get("order_count")), + ) + + +def load_book_replay_events(path: str) -> list[BookReplayEvent]: + """Load a newline-delimited JSON book-replay trace.""" + events = [] + with open(path, "r", encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if not line.strip(): + continue + try: + payload = json.loads(line) + except json.JSONDecodeError as exc: + raise BookReplayError(f"invalid JSON on line {line_number}: {exc.msg}") from exc + try: + events.append(BookReplayEvent.from_mapping(payload)) + except (TypeError, ValueError) as exc: + raise BookReplayError(f"invalid event on line {line_number}: {exc}") from exc + return events + + +def trace_sha256(events: Iterable[BookReplayEvent]) -> str: + """Hash normalized events independently of input whitespace.""" + digest = hashlib.sha256() + for event in events: + digest.update(_canonical_json(event.to_dict())) + digest.update(b"\n") + return digest.hexdigest() + + +def _canonical_json(payload: Any) -> bytes: + return json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +__all__ = [ + "ARTIFACT_SCHEMA_VERSION", + "PROFILE_NAME", + "PROTOCOL_NAME", + "PROTOCOL_VERSION", + "BookReplayConfig", + "BookReplayError", + "BookReplayEvent", + "BookReplayObservation", + "BookReplaySnapshot", + "BookReplayState", + "EngineMetadata", + "Outcome", + "RestingBookOrder", + "SimulatedFill", + "load_book_replay_events", + "trace_sha256", +] diff --git a/src/tracebook/book_replay/protocol.py b/src/tracebook/book_replay/protocol.py new file mode 100644 index 0000000..9ad8dfc --- /dev/null +++ b/src/tracebook/book_replay/protocol.py @@ -0,0 +1,134 @@ +"""NDJSON protocol helpers for external L3 book-replay adapters.""" + +from __future__ import annotations + +import json +import sys +from typing import Callable, Optional, Protocol, TextIO + +from .model import ( + PROTOCOL_NAME, + PROTOCOL_VERSION, + BookReplayConfig, + BookReplayEvent, + BookReplayObservation, + BookReplayState, + EngineMetadata, +) + + +class BookReplayAdapter(Protocol): + """Minimal interface shared by in-process and external book mirrors.""" + + metadata: EngineMetadata + + def apply(self, event: BookReplayEvent, index: int) -> BookReplayObservation: ... + + def snapshot(self) -> BookReplayState: ... + + def close(self) -> None: ... + + +BookReplayAdapterFactory = Callable[[BookReplayConfig], BookReplayAdapter] + + +def _write_message(stream: TextIO, payload: dict) -> None: + stream.write(json.dumps(payload, separators=(",", ":"), allow_nan=False) + "\n") + stream.flush() + + +def serve_book_replay_stdio( + adapter_factory: BookReplayAdapterFactory, + input_stream: Optional[TextIO] = None, + output_stream: Optional[TextIO] = None, +) -> int: + """Serve one adapter over the isolated book-replay protocol.""" + source = input_stream or sys.stdin + sink = output_stream or sys.stdout + adapter: Optional[BookReplayAdapter] = None + last_index = 0 + try: + first_line = source.readline() + if not first_line: + raise ValueError("expected hello message") + hello = json.loads(first_line) + if not isinstance(hello, dict) or hello.get("type") != "hello": + raise ValueError("first message must be hello") + if hello.get("protocol") != PROTOCOL_NAME: + raise ValueError(f"protocol must be {PROTOCOL_NAME!r}") + if hello.get("protocol_version") != PROTOCOL_VERSION: + raise ValueError(f"protocol_version must be {PROTOCOL_VERSION}") + adapter = adapter_factory(BookReplayConfig.from_dict(hello.get("config", {}))) + _write_message( + sink, + { + "type": "ready", + "protocol": PROTOCOL_NAME, + "protocol_version": PROTOCOL_VERSION, + "engine": adapter.metadata.to_dict(), + }, + ) + + for line in source: + if not line.strip(): + continue + message = json.loads(line) + if not isinstance(message, dict): + raise ValueError("protocol message must be an object") + message_type = message.get("type") + if message_type == "event": + index = message.get("index") + if isinstance(index, bool) or not isinstance(index, int) or index <= 0: + raise ValueError("event index must be a positive integer") + if index != last_index + 1: + raise ValueError("event indexes must be contiguous and start at 1") + event = BookReplayEvent.from_mapping(message.get("event", {})) + observation = adapter.apply(event, index) + if observation.index != index: + raise ValueError("adapter returned the wrong observation index") + last_index = index + _write_message(sink, observation.to_dict(include_type=True)) + elif message_type == "snapshot": + if message.get("index") != last_index: + raise ValueError("snapshot index does not match the last event") + _write_message( + sink, + { + "type": "snapshot", + "index": last_index, + "state": adapter.snapshot().to_dict(), + }, + ) + elif message_type == "finish": + if message.get("event_count") != last_index: + raise ValueError("finish event_count does not match the last event") + _write_message(sink, {"type": "complete", "event_count": last_index}) + return 0 + else: + raise ValueError(f"unsupported protocol message type: {message_type!r}") + raise ValueError("protocol ended before finish") + except (BrokenPipeError, json.JSONDecodeError, TypeError, ValueError) as exc: + try: + _write_message(sink, {"type": "error", "code": "PROTOCOL_ERROR", "message": str(exc)}) + except BrokenPipeError: + pass + return 2 + except Exception as exc: + try: + _write_message(sink, {"type": "error", "code": "ADAPTER_ERROR", "message": str(exc)}) + except BrokenPipeError: + pass + return 2 + finally: + if adapter is not None: + try: + adapter.close() + except Exception as exc: + print(f"Adapter close error: {exc}", file=sys.stderr) + + +__all__ = [ + "BookReplayAdapter", + "BookReplayAdapterFactory", + "serve_book_replay_stdio", +] diff --git a/src/tracebook/book_replay/reference.py b/src/tracebook/book_replay/reference.py new file mode 100644 index 0000000..907b966 --- /dev/null +++ b/src/tracebook/book_replay/reference.py @@ -0,0 +1,199 @@ +"""Inspectable reference semantics for L3 book replay.""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from typing import Dict, List, Tuple, cast + +from .._version import __version__ +from .model import ( + BookReplayConfig, + BookReplayError, + BookReplayEvent, + BookReplayObservation, + BookReplaySnapshot, + BookReplayState, + EngineMetadata, + Outcome, + RestingBookOrder, + SimulatedFill, + canonical_decimal, +) + + +@dataclass +class _Order: + order_id: int + side: str + price: Decimal + quantity: Decimal + + +class _Book: + def __init__(self) -> None: + self.levels: Dict[str, Dict[Decimal, List[_Order]]] = {"BUY": {}, "SELL": {}} + self.orders: Dict[int, _Order] = {} + + +class ReferenceBookReplayAdapter: + """Mirror L3 deltas and simulate fills without crossing submitted orders.""" + + def __init__( + self, + config: BookReplayConfig, + engine_name: str = "tracebook-book-replay-reference", + engine_version: str = __version__, + ) -> None: + if not isinstance(config, BookReplayConfig): + raise BookReplayError("reference adapter requires BookReplayConfig") + self.config = config + self.metadata = EngineMetadata(engine_name, engine_version, "Python") + self._books: Dict[str, _Book] = {} + + def apply(self, event: BookReplayEvent, index: int) -> BookReplayObservation: + """Apply one delta or probe and emit its canonical observation.""" + if not isinstance(event, BookReplayEvent): + raise BookReplayError("reference adapter requires BookReplayEvent values") + + fills: Tuple[SimulatedFill, ...] = () + outcome = Outcome("applied") + if event.op == "add": + outcome = self._add(event) + elif event.op == "update": + outcome = self._update(event) + elif event.op == "delete": + outcome = self._delete(event) + else: + fills = self._probe(event) + + state = self.snapshot() + return BookReplayObservation(index, outcome, fills, state.digest(), state.order_count) + + def _add(self, event: BookReplayEvent) -> Outcome: + book = self._books.setdefault(event.symbol, _Book()) + order_id = cast(int, event.order_id) + if order_id in book.orders: + return Outcome( + "rejected", + "DUPLICATE_ORDER_ID", + f"Order {order_id} is already present in {event.symbol}", + ) + side = cast(str, event.side) + price = Decimal(cast(str, event.price)) + order = _Order(order_id, side, price, Decimal(cast(str, event.quantity))) + book.orders[order_id] = order + book.levels[side].setdefault(price, []).append(order) + return Outcome("applied") + + def _update(self, event: BookReplayEvent) -> Outcome: + book = self._books.get(event.symbol) + order_id = cast(int, event.order_id) + if book is None or order_id not in book.orders: + return Outcome( + "rejected", + "ORDER_NOT_ACTIVE", + f"Order {order_id} is not present in {event.symbol}", + ) + + order = book.orders[order_id] + new_side = cast(str, event.side) + new_price = Decimal(cast(str, event.price)) + new_quantity = Decimal(cast(str, event.quantity)) + if order.side == new_side and order.price == new_price: + # A mirrored update does not decide venue priority. Like NautilusTrader's + # BookLevel update, replacing the value under the same key keeps its slot. + order.quantity = new_quantity + return Outcome("applied") + + self._remove_from_level(book, order) + order.side = new_side + order.price = new_price + order.quantity = new_quantity + book.levels[new_side].setdefault(new_price, []).append(order) + return Outcome("applied") + + def _delete(self, event: BookReplayEvent) -> Outcome: + book = self._books.get(event.symbol) + order_id = cast(int, event.order_id) + if book is None or order_id not in book.orders: + return Outcome( + "rejected", + "ORDER_NOT_ACTIVE", + f"Order {order_id} is not present in {event.symbol}", + ) + order = book.orders.pop(order_id) + self._remove_from_level(book, order) + return Outcome("applied") + + @staticmethod + def _remove_from_level(book: _Book, order: _Order) -> None: + levels = book.levels[order.side] + level = levels[order.price] + level.remove(order) + if not level: + del levels[order.price] + + def _probe(self, event: BookReplayEvent) -> Tuple[SimulatedFill, ...]: + book = self._books.get(event.symbol) + if book is None: + return () + + incoming_side = cast(str, event.side) + resting_side = "SELL" if incoming_side == "BUY" else "BUY" + limit_price = Decimal(cast(str, event.price)) + remaining = Decimal(cast(str, event.quantity)) + levels = book.levels[resting_side] + prices = sorted(levels, reverse=resting_side == "BUY") + fills = [] + for price in prices: + if incoming_side == "BUY" and price > limit_price: + break + if incoming_side == "SELL" and price < limit_price: + break + for order in levels[price]: + quantity = min(order.quantity, remaining) + fills.append( + SimulatedFill( + price=canonical_decimal(price), + quantity=canonical_decimal(quantity), + ) + ) + remaining -= quantity + if remaining == 0: + return tuple(fills) + return tuple(fills) + + def snapshot(self) -> BookReplayState: + """Return every mirrored order in price-time priority order.""" + snapshots = [] + for symbol in sorted(self._books): + book = self._books[symbol] + snapshots.append( + BookReplaySnapshot( + symbol=symbol, + bids=self._snapshot_side(book, "BUY"), + asks=self._snapshot_side(book, "SELL"), + ) + ) + return BookReplayState(tuple(snapshots)) + + @staticmethod + def _snapshot_side(book: _Book, side: str) -> Tuple[RestingBookOrder, ...]: + orders = [] + for price in sorted(book.levels[side], reverse=side == "BUY"): + for order in book.levels[side][price]: + orders.append( + RestingBookOrder( + order_id=order.order_id, + price=canonical_decimal(order.price), + quantity=canonical_decimal(order.quantity), + ) + ) + return tuple(orders) + + def close(self) -> None: + """Reference adapters own no external resources.""" + + +__all__ = ["ReferenceBookReplayAdapter"] diff --git a/tests/test_book_replay.py b/tests/test_book_replay.py new file mode 100644 index 0000000..eaf79e0 --- /dev/null +++ b/tests/test_book_replay.py @@ -0,0 +1,283 @@ +import io +import json +import sys +from pathlib import Path + +import pytest + +from tracebook.book_replay import ( + PROFILE_NAME, + PROTOCOL_NAME, + PROTOCOL_VERSION, + BookReplayConfig, + BookReplayError, + BookReplayEvent, + BookReplayObservation, + BookReplaySnapshot, + BookReplayState, + EngineMetadata, + ExternalBookReplayAdapter, + ReferenceBookReplayAdapter, + run_book_replay, + serve_book_replay_stdio, +) +from tracebook.book_replay.cli import main + +ROOT = Path(__file__).parents[1] +PROFILE_TRACE = ROOT / "src" / "tracebook" / "book_replay" / "fixtures" / f"{PROFILE_NAME}.jsonl" +EXAMPLE_ADAPTER = ROOT / "examples" / "book_replay_adapter.py" + + +def _event(op="add", **overrides): + values = { + "op": op, + "symbol": "TEST", + "order_id": 1, + "side": "BUY", + "price": "100", + "quantity": "2", + } + if op == "delete": + values = {"op": op, "symbol": "TEST", "order_id": 1} + elif op == "probe": + values = { + "op": op, + "symbol": "TEST", + "side": "SELL", + "price": "100", + "quantity": "2", + } + values.update(overrides) + return BookReplayEvent.from_mapping(values) + + +def test_event_contract_is_canonical_and_strict(): + event = _event(price="100.00", quantity=2.500) + + assert event.to_dict() == { + "op": "add", + "symbol": "TEST", + "order_id": 1, + "side": "BUY", + "price": "100", + "quantity": "2.5", + } + with pytest.raises(BookReplayError, match="unsupported fields"): + BookReplayEvent.from_mapping({**event.to_dict(), "owner": 9}) + with pytest.raises(BookReplayError, match="contain only"): + BookReplayEvent("delete", "TEST", 1, "BUY") + with pytest.raises(BookReplayError, match="must be positive"): + _event(quantity="0") + + +def test_same_level_size_increase_retains_fifo_and_probe_does_not_mutate(): + adapter = ReferenceBookReplayAdapter(BookReplayConfig()) + adapter.apply(_event(order_id=1, quantity="2"), 1) + adapter.apply(_event(order_id=2, quantity="3"), 2) + updated = adapter.apply(_event("update", order_id=1, quantity="4"), 3) + before_probe = adapter.snapshot() + + observation = adapter.apply(_event("probe", quantity="6"), 4) + + assert [order.order_id for order in before_probe.books[0].bids] == [1, 2] + assert [fill.to_dict() for fill in observation.fills] == [ + {"price": "100", "quantity": "4"}, + {"price": "100", "quantity": "2"}, + ] + assert adapter.snapshot() == before_probe + assert observation.state_hash == updated.state_hash + + +def test_price_change_requeues_and_delete_is_keyed_by_order_id(): + adapter = ReferenceBookReplayAdapter(BookReplayConfig()) + adapter.apply(_event(order_id=1, price="99"), 1) + adapter.apply(_event(order_id=2, price="99"), 2) + adapter.apply(_event("update", order_id=1, price="100", quantity="2"), 3) + adapter.apply(_event("update", order_id=1, price="99", quantity="2"), 4) + + assert [order.order_id for order in adapter.snapshot().books[0].bids] == [2, 1] + deleted = adapter.apply(_event("delete", order_id=2), 5) + rejected = adapter.apply(_event("delete", order_id=999), 6) + assert deleted.outcome.status == "applied" + assert rejected.outcome.to_dict()["reason"] == "ORDER_NOT_ACTIVE" + + +def test_probe_obeys_limit_price_and_price_time_order(): + adapter = ReferenceBookReplayAdapter(BookReplayConfig()) + events = [ + _event(order_id=10, side="SELL", price="102", quantity="5"), + _event(order_id=11, side="SELL", price="101", quantity="1"), + _event(order_id=12, side="SELL", price="101", quantity="2"), + ] + for index, event in enumerate(events, 1): + adapter.apply(event, index) + + observation = adapter.apply( + _event("probe", side="BUY", price="101", quantity="5"), + 4, + ) + + assert [fill.to_dict() for fill in observation.fills] == [ + {"price": "101", "quantity": "1"}, + {"price": "101", "quantity": "2"}, + ] + assert observation.order_count == 3 + + +class _ReorderingAdapter: + def __init__(self, config): + self._inner = ReferenceBookReplayAdapter(config) + self.metadata = EngineMetadata("reordering-test-adapter", "1", "Python") + self._reorder = False + + def apply(self, event, index): + observation = self._inner.apply(event, index) + if event.op == "update" and event.order_id == 1: + self._reorder = True + state = self.snapshot() + return BookReplayObservation( + index, + observation.outcome, + observation.fills, + state.digest(), + state.order_count, + ) + + def snapshot(self): + state = self._inner.snapshot() + if not self._reorder: + return state + books = [] + for book in state.books: + bids = list(book.bids) + if book.symbol == "TEST" and len(bids) >= 2: + bids[0], bids[1] = bids[1], bids[0] + books.append(BookReplaySnapshot(book.symbol, tuple(bids), book.asks)) + return BookReplayState(tuple(books)) + + def close(self): + self._inner.close() + + +def test_report_localizes_same_level_requeue_as_book_state_divergence(): + events = [ + _event(order_id=1), + _event(order_id=2), + _event("update", order_id=1, quantity="4"), + ] + + report = run_book_replay(events, _ReorderingAdapter, trace_name="priority") + + assert report.conformant is False + assert report.compared_events == 3 + assert report.divergence is not None + assert report.divergence.category == "book_state" + assert report.divergence.path.endswith("order_id") + assert report.to_dict()["artifact_type"] == "tracebook.book-replay.report" + assert report.to_dict()["config"] == {"profile": PROFILE_NAME} + + +def test_reference_candidate_conforms_on_bundled_profile(): + events = [ + BookReplayEvent.from_mapping(json.loads(line)) + for line in PROFILE_TRACE.read_text(encoding="utf-8").splitlines() + ] + + report = run_book_replay(events, ReferenceBookReplayAdapter, trace_name=PROFILE_NAME) + + assert report.conformant is True + assert report.event_count == 17 + assert report.compared_events == 17 + assert ( + report.final_state_hash + == "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002" + ) + + +def test_stdio_transcript_uses_separate_protocol_identity(): + event = _event() + messages = [ + { + "type": "hello", + "protocol": PROTOCOL_NAME, + "protocol_version": PROTOCOL_VERSION, + "config": BookReplayConfig().to_dict(), + }, + {"type": "event", "index": 1, "event": event.to_dict()}, + {"type": "snapshot", "index": 1}, + {"type": "finish", "event_count": 1}, + ] + source = io.StringIO("".join(json.dumps(message) + "\n" for message in messages)) + sink = io.StringIO() + + assert serve_book_replay_stdio(ReferenceBookReplayAdapter, source, sink) == 0 + responses = [json.loads(line) for line in sink.getvalue().splitlines()] + assert [response["type"] for response in responses] == [ + "ready", + "observation", + "snapshot", + "complete", + ] + assert responses[0]["protocol"] == "tracebook.book-replay" + assert responses[2]["state"]["books"][0]["bids"][0]["order_id"] == 1 + + +def test_external_example_adapter_conforms(): + events = [_event(order_id=1), _event("probe", quantity="1")] + + report = run_book_replay( + events, + lambda config: ExternalBookReplayAdapter( + [sys.executable, str(EXAMPLE_ADAPTER)], + config, + timeout_seconds=2, + ), + ) + + assert report.conformant is True + assert report.candidate_engine.name == "example-book-replay-adapter" + + +def test_cli_sample_and_external_run(tmp_path, capsys): + sample_dir = tmp_path / "sample" + assert main(["sample", str(sample_dir)]) == 0 + copied = sample_dir / f"{PROFILE_NAME}.jsonl" + assert copied.read_bytes() == PROFILE_TRACE.read_bytes() + + output = tmp_path / "report.json" + assert ( + main( + [ + "run", + str(copied), + "--output", + str(output), + "--candidate-cmd", + f"{sys.executable} {EXAMPLE_ADAPTER}", + ] + ) + == 0 + ) + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["conformant"] is True + assert payload["artifact_type"] == "tracebook.book-replay.report" + assert "Book-replay trace copied" in capsys.readouterr().out + + +def test_cli_rejects_overwriting_input(tmp_path, capsys): + trace = tmp_path / "trace.jsonl" + trace.write_text(json.dumps(_event().to_dict()) + "\n", encoding="utf-8") + + exit_code = main( + [ + "run", + str(trace), + "--output", + str(trace), + "--candidate-cmd", + f"{sys.executable} {EXAMPLE_ADAPTER}", + ] + ) + + assert exit_code == 2 + assert "input and output paths must be distinct" in capsys.readouterr().err diff --git a/tests/test_external_integrations.py b/tests/test_external_integrations.py index b56816a..7a9ceee 100644 --- a/tests/test_external_integrations.py +++ b/tests/test_external_integrations.py @@ -1,3 +1,4 @@ +import importlib.util import sys from pathlib import Path @@ -14,6 +15,17 @@ UPSTREAM_REPOSITORY, PythonMatchingEngineAdapter, ) +from integrations.nautilus_trader.adapter import ( + UPSTREAM_COMMIT as NAUTILUS_COMMIT, + UPSTREAM_REPOSITORY as NAUTILUS_REPOSITORY, + UPSTREAM_TAG as NAUTILUS_TAG, + UPSTREAM_VERSION as NAUTILUS_VERSION, +) +from tracebook.book_replay import ( + ExternalBookReplayAdapter, + load_book_replay_events, + run_book_replay, +) from tracebook.conformance import ( AdapterProtocolError, ConformanceConfig, @@ -27,6 +39,7 @@ INTEGRATION = ROOT / "integrations" / "python_matching_engine" RUST_INTEGRATION = ROOT / "integrations" / "orderbook_rs" GOCRONX_INTEGRATION = ROOT / "integrations" / "gocronx_matcher" +NAUTILUS_INTEGRATION = ROOT / "integrations" / "nautilus_trader" RUST_PROTOCOL = ROOT / "integrations" / "rust_protocol" @@ -59,6 +72,49 @@ def test_python_matching_engine_adapter_explains_a_missing_upstream(monkeypatch, ) +def test_nautilus_book_replay_integration_is_version_and_source_pinned(): + readme = (NAUTILUS_INTEGRATION / "README.md").read_text(encoding="utf-8") + workflow = (ROOT / ".github" / "workflows" / "nautilus-trader-book-replay.yml").read_text( + encoding="utf-8" + ) + + assert NAUTILUS_REPOSITORY == "https://github.com/nautechsystems/nautilus_trader.git" + assert NAUTILUS_TAG == "v2.0.0rc3" + assert NAUTILUS_COMMIT == "648970ce64a304d93da0a29320cb6e19b905fa39" + assert NAUTILUS_VERSION == "2.0.0rc3" + assert "LGPL-3.0-only" in readme + assert "does not claim" in readme + assert '"nautilus-trader==2.0.0rc3"' in workflow + assert "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002" in workflow + + +@pytest.mark.skipif( + importlib.util.find_spec("nautilus_trader") is None, + reason="the optional pinned NautilusTrader runtime is not installed", +) +def test_pinned_nautilus_runtime_conforms_to_book_replay_profile(): + events = load_book_replay_events( + ROOT / "src" / "tracebook" / "book_replay" / "fixtures" / "l3-book-replay-v1.jsonl" + ) + + report = run_book_replay( + events, + lambda config: ExternalBookReplayAdapter( + [sys.executable, str(NAUTILUS_INTEGRATION / "adapter.py")], + config, + timeout_seconds=10, + ), + ) + + assert report.conformant is True + assert report.compared_events == 17 + assert report.candidate_engine.version == NAUTILUS_VERSION + assert ( + report.final_state_hash + == "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002" + ) + + def test_orderbook_rs_integration_pins_engine_toolchain_and_dependency_graph(): cargo = (RUST_INTEGRATION / "Cargo.toml").read_text(encoding="utf-8") lock = (RUST_INTEGRATION / "Cargo.lock").read_text(encoding="utf-8") @@ -162,6 +218,7 @@ def test_source_manifest_includes_native_integration_files(): assert "recursive-include integrations/orderbook_rs/src *.rs" in manifest assert package_data["tracebook.conformance.fixtures.v2"] == ["*.json", "*.jsonl"] assert "recursive-include integrations *.py *.md *.json *.jsonl" in manifest + assert "recursive-include src/tracebook/book_replay/fixtures *.jsonl" in manifest assert "include integrations/orderbook_rs/Cargo.lock" in manifest assert "prune integrations/orderbook_rs/target" in manifest assert "recursive-include integrations/gocronx_matcher/src *.rs" in manifest diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 4a456f6..8545cbf 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -47,6 +47,7 @@ def test_distribution_name_cli_and_typing_metadata_are_release_ready(): } assert package_data["tracebook.corpus.fixtures"] == ["coinbase-btcusd-synthetic-v1/*"] assert package_data["tracebook.conformance.fixtures.v1"] == ["*.json", "*.jsonl"] + assert package_data["tracebook.book_replay.fixtures"] == ["*.jsonl"] assert (ROOT / "src" / "tracebook" / "py.typed").is_file() assert (ROOT / ".github" / "workflows" / "release.yml").is_file() diff --git a/tools/smoke_conformance_wheel.py b/tools/smoke_conformance_wheel.py index 620e243..323e993 100644 --- a/tools/smoke_conformance_wheel.py +++ b/tools/smoke_conformance_wheel.py @@ -51,9 +51,14 @@ def _assert_lightweight_environment(python: Path) -> None: raise SystemExit(f"{package} unexpectedly installed in isolated environment") import tracebook.conformance +import tracebook.book_replay import importlib.metadata +from importlib import resources assert importlib.metadata.version("tracebook-conformance") == tracebook.__version__ +assert resources.files("tracebook.book_replay.fixtures").joinpath( + "l3-book-replay-v1.jsonl" +).is_file() """ _run([str(python), "-c", probe]) From e5cbf0a234b1d2a62a93e5724025802f0f01d8f2 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Mon, 31 Aug 2026 06:54:22 -0400 Subject: [PATCH 12/19] Add generated L3 failure reduction --- .../workflows/nautilus-trader-book-replay.yml | 85 ++- docs/book-replay-conformance.md | 44 +- integrations/nautilus_trader/README.md | 23 + integrations/nautilus_trader/adapter.py | 18 +- .../nautilus_trader/faulty_adapter.py | 41 ++ .../regressions/upsize-requeue-failure.json | 38 ++ .../regressions/upsize-requeue-reduced.jsonl | 3 + src/tracebook/book_replay/__init__.py | 23 + src/tracebook/book_replay/campaign.py | 533 ++++++++++++++++++ src/tracebook/book_replay/cli.py | 98 +++- src/tracebook/book_replay/minimize.py | 172 ++++++ tests/fixtures/faulty_book_replay_adapter.py | 54 ++ tests/test_book_replay_campaign.py | 259 +++++++++ tests/test_external_integrations.py | 5 + 14 files changed, 1376 insertions(+), 20 deletions(-) create mode 100644 integrations/nautilus_trader/faulty_adapter.py create mode 100644 integrations/nautilus_trader/regressions/upsize-requeue-failure.json create mode 100644 integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl create mode 100644 src/tracebook/book_replay/campaign.py create mode 100644 src/tracebook/book_replay/minimize.py create mode 100644 tests/fixtures/faulty_book_replay_adapter.py create mode 100644 tests/test_book_replay_campaign.py diff --git a/.github/workflows/nautilus-trader-book-replay.yml b/.github/workflows/nautilus-trader-book-replay.yml index b34e6e9..64b6437 100644 --- a/.github/workflows/nautilus-trader-book-replay.yml +++ b/.github/workflows/nautilus-trader-book-replay.yml @@ -9,7 +9,8 @@ on: - ".github/workflows/nautilus-trader-book-replay.yml" - "integrations/nautilus_trader/**" - "src/tracebook/book_replay/**" - - "tests/test_book_replay.py" + - "tests/test_book_replay*.py" + - "tests/fixtures/faulty_book_replay_adapter.py" - "tests/test_external_integrations.py" permissions: @@ -42,12 +43,90 @@ jobs: --candidate python integrations/nautilus_trader/adapter.py python -c "import json; p=json.load(open('/tmp/nautilus-book-replay.json')); assert p['artifact_type'] == 'tracebook.book-replay.report'; assert p['conformant'] is True and p['compared_events'] == 17; assert p['candidate_engine']['version'] == '2.0.0rc3'; assert p['final_state_hash'] == '9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002'" + - name: Run the deterministic generated campaign + run: | + python -m tracebook.book_replay campaign \ + --output /tmp/nautilus-book-campaign.json \ + --seed 20260831 \ + --traces 25 \ + --events-per-trace 100 \ + --timeout 10 \ + --candidate python -m integrations.nautilus_trader.adapter + python - <<'PY' + import json + + with open("/tmp/nautilus-book-campaign.json", encoding="utf-8") as handle: + payload = json.load(handle) + assert payload["campaign_id"] == "sha256:09b1599eaeb474d98617acc7869ace26759ed5ab8350803f06197ef572864bab" + assert payload["conformant"] is True + assert payload["completed_traces"] == 25 + assert payload["generated_events"] == 2500 + assert payload["semantic_coverage"]["complete"] is True + assert payload["semantic_coverage"]["covered"] == 11 + PY + + - name: Prove the harness catches and reduces a native queue-priority fault + run: | + set +e + python -m tracebook.book_replay campaign \ + --output /tmp/nautilus-negative-control.json \ + --reduced-events-output /tmp/nautilus-negative-control.jsonl \ + --seed 20260831 \ + --traces 10 \ + --events-per-trace 100 \ + --timeout 10 \ + --candidate python -m integrations.nautilus_trader.faulty_adapter + status=$? + set -e + test "$status" -eq 1 + cmp \ + integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl \ + /tmp/nautilus-negative-control.jsonl + python - <<'PY' + import json + + with open("/tmp/nautilus-negative-control.json", encoding="utf-8") as handle: + payload = json.load(handle) + with open( + "integrations/nautilus_trader/regressions/upsize-requeue-failure.json", + encoding="utf-8", + ) as handle: + expected = json.load(handle) + failure = payload["failure"] + actual = { + "artifact_type": "tracebook.book-replay.negative-control", + "budget_exhausted": failure["budget_exhausted"], + "campaign_id": payload["campaign_id"], + "candidate_engine": payload["candidate_engine"], + "failure_class": failure["failure_class"], + "failure_id": failure["failure_id"], + "minimized_event_count": failure["minimized_event_count"], + "minimized_trace_sha256": failure["minimized_trace_sha256"], + "one_minimal": failure["one_minimal"], + "original_divergence_event": failure["original_divergence_event"], + "original_event_count": failure["original_event_count"], + "original_trace_sha256": failure["original_trace_sha256"], + "reduced_divergence": failure["minimization"]["book_replay_report"]["divergence"], + "schema_version": payload["schema_version"], + "trace_index": failure["trace_index"], + } + assert actual == expected + PY + - name: Run the book-replay and integration contract tests - run: python -m pytest tests/test_book_replay.py tests/test_external_integrations.py + run: >- + python -m pytest + tests/test_book_replay.py + tests/test_book_replay_campaign.py + tests/test_external_integrations.py - uses: actions/upload-artifact@v7 if: always() with: name: nautilus-trader-book-replay - path: /tmp/nautilus-book-replay.json + path: | + /tmp/nautilus-book-replay.json + /tmp/nautilus-book-campaign.json + /tmp/nautilus-negative-control.json + /tmp/nautilus-negative-control.jsonl if-no-files-found: error diff --git a/docs/book-replay-conformance.md b/docs/book-replay-conformance.md index 511ffef..db55b92 100644 --- a/docs/book-replay-conformance.md +++ b/docs/book-replay-conformance.md @@ -95,6 +95,47 @@ python -m tracebook.book_replay run \ Exit `0` means semantic agreement, `1` means the first semantic divergence was recorded, and `2` means input, process, or protocol failure. +## Generated Campaigns And Reduction + +The v1 generator uses a specified SplitMix64 stream rather than Python's +runtime PRNG. Each trace draws its prefix from an 18-event semantic scaffold; +at 18 events or longer it reaches all 11 declared capabilities, then continues +with state-aware generated adds, updates, deletes, and probes. Candidate +behavior never feeds back into event generation. + +Run a deterministic campaign: + +```bash +python -m tracebook.book_replay campaign \ + --output /tmp/book-replay-campaign.json \ + --reduced-events-output /tmp/book-replay-reduced.jsonl \ + --seed 20260831 \ + --traces 25 \ + --events-per-trace 100 \ + --candidate-cmd './book-adapter --tracebook-book-replay-stdio' +``` + +The runner stops at the first divergence and automatically delta-debugs its +compared prefix. Reduction preserves the first-divergence category, rejects a +new protocol failure while reducing a semantic failure, records its candidate +run budget, and reports whether the result is one-minimal. Campaign artifacts +include the generator version, campaign ID, per-trace seeds and hashes, +capability coverage, original evidence, minimized evidence, and a stable +failure ID. + +An already captured divergent trace can be minimized directly: + +```bash +python -m tracebook.book_replay minimize trace.jsonl \ + --events-output /tmp/reduced.jsonl \ + --output /tmp/minimization.json \ + --candidate-cmd './book-adapter --tracebook-book-replay-stdio' +``` + +`campaign` artifacts use `artifact_type = "tracebook.book-replay.campaign"`; +standalone reductions use +`artifact_type = "tracebook.book-replay.minimization"`. + Python adapters can use the server helper: ```python @@ -110,7 +151,8 @@ The complete reference-backed example is The maintained optional adapter targets NautilusTrader's native Rust-backed `L3_MBO` book and direct `simulate_fills(BookOrder)` Python binding. It passes -all 17 bundled events at the pinned `2.0.0rc3` release candidate. See the +all 17 bundled events and a pinned 2,500-event generated campaign at the +`2.0.0rc3` release candidate. See the [`integrations/nautilus_trader`](../integrations/nautilus_trader) proof, provenance, exact command, and LGPL distribution warning. diff --git a/integrations/nautilus_trader/README.md b/integrations/nautilus_trader/README.md index 650188b..6beba86 100644 --- a/integrations/nautilus_trader/README.md +++ b/integrations/nautilus_trader/README.md @@ -39,6 +39,11 @@ The 17-event bundled trace covers: No trace order is submitted to NautilusTrader's execution engine. The only liquidity is the exogenous L3 state built from deltas. +The scheduled proof also runs generator version 1 with seed `20260831`: 25 +independent traces of 100 events each. The stable campaign ID is +`sha256:09b1599eaeb474d98617acc7869ace26759ed5ab8350803f06197ef572864bab`. +It reaches all 11 declared semantic capabilities and passes all 2,500 events. + ## Run The Pinned Proof Use a separate Python 3.12-3.14 environment because the v2 candidate does not @@ -71,6 +76,24 @@ The command exits `0` and the pinned candidate produces: } ``` +## Negative Control And Retained Evidence + +[`faulty_adapter.py`](faulty_adapter.py) subclasses the real pinned native +adapter and injects one deliberate fault: a same-price size increase is +implemented as delete-plus-add, moving the order to the back of its level. +This file is a harness negative control, not a candidate integration. + +The generated campaign must exit `1`, localize the first mismatch to +`$.state.books[0].bids[0].order_id`, and reduce it to the three events in +[`regressions/upsize-requeue-reduced.jsonl`](regressions/upsize-requeue-reduced.jsonl). +The retained metadata in +[`regressions/upsize-requeue-failure.json`](regressions/upsize-requeue-failure.json) +pins failure ID `failure-dfe5c23848b63211b655`, the reduced trace hash, and +the expected localized divergence. CI regenerates both forms and compares them +to the retained evidence. This proves the test surface can reject a known +native queue-priority defect; it does not imply the unmodified upstream has +that defect. + ## License Boundary NautilusTrader declares `LGPL-3.0-only` in its Cargo workspace. It is not a diff --git a/integrations/nautilus_trader/adapter.py b/integrations/nautilus_trader/adapter.py index a903705..38bef21 100644 --- a/integrations/nautilus_trader/adapter.py +++ b/integrations/nautilus_trader/adapter.py @@ -103,13 +103,7 @@ def apply(self, event, index): book = self._books[event.symbol] old_order = known_orders[event.order_id] native_order = self._native_order(event) - if old_order.side != native_order.side: - # Native OrderBook.update selects one ladder from the new side. A - # normalized side move is therefore delete+add at this boundary. - book.delete(old_order, 0, index, index) - book.add(native_order, 0, index, index) - else: - book.update(native_order, 0, index, index) + self._apply_native_update(book, old_order, native_order, index) known_orders[event.order_id] = native_order elif event.op == "delete": if event.order_id not in known_orders: @@ -142,6 +136,16 @@ def apply(self, event, index): order_count=state.order_count, ) + @staticmethod + def _apply_native_update(book, old_order, native_order, index): + if old_order.side != native_order.side: + # Native OrderBook.update selects one ladder from the new side. A + # normalized side move is therefore delete+add at this boundary. + book.delete(old_order, 0, index, index) + book.add(native_order, 0, index, index) + else: + book.update(native_order, 0, index, index) + def snapshot(self): snapshots = [] for symbol in sorted(self._books): diff --git a/integrations/nautilus_trader/faulty_adapter.py b/integrations/nautilus_trader/faulty_adapter.py new file mode 100644 index 0000000..9d0de16 --- /dev/null +++ b/integrations/nautilus_trader/faulty_adapter.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Negative control that deliberately requeues same-level Nautilus upsizes.""" + +from __future__ import annotations + +from decimal import Decimal + +from tracebook.book_replay import EngineMetadata, serve_book_replay_stdio + +from integrations.nautilus_trader.adapter import NautilusTraderBookReplayAdapter + + +class FaultyNautilusTraderBookReplayAdapter(NautilusTraderBookReplayAdapter): + """Inject one documented queue-priority fault into the real native book.""" + + def __init__(self, config): + super().__init__(config) + self.metadata = EngineMetadata( + "NautilusTrader L3 negative control: upsize requeue", + self.metadata.version, + self.metadata.language, + ) + + @staticmethod + def _apply_native_update(book, old_order, native_order, index): + same_level = old_order.side == native_order.side and old_order.price == native_order.price + is_upsize = Decimal(str(native_order.size)) > Decimal(str(old_order.size)) + if same_level and is_upsize: + book.delete(old_order, 0, index, index) + book.add(native_order, 0, index, index) + return + NautilusTraderBookReplayAdapter._apply_native_update( + book, + old_order, + native_order, + index, + ) + + +if __name__ == "__main__": + raise SystemExit(serve_book_replay_stdio(FaultyNautilusTraderBookReplayAdapter)) diff --git a/integrations/nautilus_trader/regressions/upsize-requeue-failure.json b/integrations/nautilus_trader/regressions/upsize-requeue-failure.json new file mode 100644 index 0000000..4df73b6 --- /dev/null +++ b/integrations/nautilus_trader/regressions/upsize-requeue-failure.json @@ -0,0 +1,38 @@ +{ + "artifact_type": "tracebook.book-replay.negative-control", + "budget_exhausted": false, + "campaign_id": "sha256:983b4500cf7665fddb7f20d3eb5b3b53817e5852dee3eac68ae37355cde39e22", + "candidate_engine": { + "language": "Rust/Python", + "name": "NautilusTrader L3 negative control: upsize requeue", + "version": "2.0.0rc3" + }, + "failure_class": "book_state:value_mismatch", + "failure_id": "failure-dfe5c23848b63211b655", + "minimized_event_count": 3, + "minimized_trace_sha256": "d9eeb771c7c469c0762c67be7dc76a63a09f36d46c1dd2cda1dbb3cbb47c0b29", + "one_minimal": true, + "original_divergence_event": 3, + "original_event_count": 3, + "original_trace_sha256": "d9eeb771c7c469c0762c67be7dc76a63a09f36d46c1dd2cda1dbb3cbb47c0b29", + "reduced_divergence": { + "candidate": 2, + "category": "book_state", + "event": { + "op": "update", + "order_id": 1, + "price": "100", + "quantity": "4", + "side": "BUY", + "symbol": "TEST" + }, + "event_index": 3, + "kind": "value_mismatch", + "message": "candidate L3 orders or queue priority differ from the reference", + "path": "$.state.books[0].bids[0].order_id", + "reference": 1, + "snapshot_error": null + }, + "schema_version": 1, + "trace_index": 1 +} diff --git a/integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl b/integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl new file mode 100644 index 0000000..cd2597e --- /dev/null +++ b/integrations/nautilus_trader/regressions/upsize-requeue-reduced.jsonl @@ -0,0 +1,3 @@ +{"op":"add","order_id":1,"price":"100","quantity":"2","side":"BUY","symbol":"TEST"} +{"op":"add","order_id":2,"price":"100","quantity":"3","side":"BUY","symbol":"TEST"} +{"op":"update","order_id":1,"price":"100","quantity":"4","side":"BUY","symbol":"TEST"} diff --git a/src/tracebook/book_replay/__init__.py b/src/tracebook/book_replay/__init__.py index 0a53d52..b17fd36 100644 --- a/src/tracebook/book_replay/__init__.py +++ b/src/tracebook/book_replay/__init__.py @@ -1,5 +1,16 @@ """A separate, explicitly non-CLOB L3 book-replay conformance surface.""" +from .campaign import ( + BOOK_REPLAY_CAPABILITIES, + BOOK_REPLAY_GENERATOR_VERSION, + BookReplayCampaignFailure, + BookReplayCampaignResult, + BookReplayCampaignTrace, + BookReplayCoverage, + generate_book_replay_trace, + measure_book_replay_coverage, + run_book_replay_campaign, +) from .compare import BookReplayDivergence, BookReplayReport, run_book_replay from .external import ( BookReplayProtocolError, @@ -23,21 +34,29 @@ SimulatedFill, load_book_replay_events, ) +from .minimize import BookReplayMinimizationResult, minimize_book_replay_failure from .protocol import BookReplayAdapter, BookReplayAdapterFactory, serve_book_replay_stdio from .reference import ReferenceBookReplayAdapter __all__ = [ "ARTIFACT_SCHEMA_VERSION", + "BOOK_REPLAY_CAPABILITIES", + "BOOK_REPLAY_GENERATOR_VERSION", "PROFILE_NAME", "PROTOCOL_NAME", "PROTOCOL_VERSION", "BookReplayAdapter", "BookReplayAdapterFactory", + "BookReplayCampaignFailure", + "BookReplayCampaignResult", + "BookReplayCampaignTrace", "BookReplayConfig", + "BookReplayCoverage", "BookReplayDivergence", "BookReplayError", "BookReplayEvent", "BookReplayObservation", + "BookReplayMinimizationResult", "BookReplayProtocolError", "BookReplayReport", "BookReplaySnapshot", @@ -50,6 +69,10 @@ "RestingBookOrder", "SimulatedFill", "load_book_replay_events", + "generate_book_replay_trace", + "measure_book_replay_coverage", + "minimize_book_replay_failure", "run_book_replay", + "run_book_replay_campaign", "serve_book_replay_stdio", ] diff --git a/src/tracebook/book_replay/campaign.py b/src/tracebook/book_replay/campaign.py new file mode 100644 index 0000000..c696658 --- /dev/null +++ b/src/tracebook/book_replay/campaign.py @@ -0,0 +1,533 @@ +"""Deterministic stateful campaigns for L3 book-replay adapters.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, replace +from decimal import Decimal +from numbers import Integral +from typing import Dict, List, Optional, Sequence, Tuple, TypeVar + +from .compare import BookReplayReport, run_book_replay +from .minimize import BookReplayMinimizationResult, minimize_book_replay_failure +from .model import ( + ARTIFACT_SCHEMA_VERSION, + PROFILE_NAME, + BookReplayConfig, + BookReplayError, + BookReplayEvent, + BookReplayState, + EngineMetadata, + trace_sha256, +) +from .protocol import BookReplayAdapterFactory +from .reference import ReferenceBookReplayAdapter + +BOOK_REPLAY_GENERATOR_VERSION = 1 +BOOK_REPLAY_CAPABILITIES = ( + "l3-add", + "same-level-fifo", + "same-level-upsize", + "price-relocation", + "l3-delete", + "duplicate-active-order-id", + "inactive-lifecycle-request", + "buy-fill-probe", + "sell-fill-probe", + "multi-level-depth", + "multiple-symbols", +) +_MASK_64 = (1 << 64) - 1 +_GOLDEN_GAMMA = 0x9E3779B97F4A7C15 +_CHOICE = TypeVar("_CHOICE") + + +class _SplitMix64: + """Specified PRNG whose output is independent of Python's random module.""" + + def __init__(self, seed: int) -> None: + self._state = seed & _MASK_64 + + def next_u64(self) -> int: + self._state = (self._state + _GOLDEN_GAMMA) & _MASK_64 + value = self._state + value = ((value ^ (value >> 30)) * 0xBF58476D1CE4E5B9) & _MASK_64 + value = ((value ^ (value >> 27)) * 0x94D049BB133111EB) & _MASK_64 + return (value ^ (value >> 31)) & _MASK_64 + + def randbelow(self, bound: int) -> int: + if bound <= 0: + raise ValueError("bound must be positive") + return self.next_u64() % bound + + def choice(self, values: Sequence[_CHOICE]) -> _CHOICE: + if not values: + raise ValueError("cannot choose from an empty sequence") + return values[self.randbelow(len(values))] + + +def _positive_int(value: int, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, Integral) or value <= 0: + raise BookReplayError(f"{field_name} must be a positive integer") + return int(value) + + +def _campaign_seed(value: int) -> int: + if isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value <= _MASK_64: + raise BookReplayError("seed must be an integer between 0 and 2^64-1") + return int(value) + + +def _trace_seed(seed: int, trace_index: int) -> int: + mixer = _SplitMix64(seed ^ ((trace_index * _GOLDEN_GAMMA) & _MASK_64)) + return mixer.next_u64() + + +def _semantic_scaffold() -> Tuple[BookReplayEvent, ...]: + """Guarantee every v1 capability before random exploration starts.""" + rows = ( + ("add", "TEST", 1, "BUY", "100", "2"), + ("add", "TEST", 2, "BUY", "100", "3"), + ("update", "TEST", 1, "BUY", "100", "4"), + ("add", "TEST", 3, "BUY", "99", "5"), + ("add", "TEST", 10, "SELL", "101", "1.5"), + ("add", "TEST", 11, "SELL", "101", "2.5"), + ("add", "TEST", 12, "SELL", "102", "4"), + ("probe", "TEST", None, "BUY", "102", "5"), + ("probe", "TEST", None, "SELL", "99", "8"), + ("update", "TEST", 2, "BUY", "99", "3"), + ("probe", "TEST", None, "SELL", "99", "12"), + ("delete", "TEST", 1, None, None, None), + ("delete", "TEST", 10, None, None, None), + ("probe", "TEST", None, "BUY", "101", "3"), + ("add", "ALT", 1, "SELL", "50", "7"), + ("probe", "ALT", None, "BUY", "50", "2"), + ("delete", "TEST", 999, None, None, None), + ("add", "TEST", 2, "BUY", "99", "1"), + ) + return tuple( + BookReplayEvent( + op=op, + symbol=symbol, + order_id=order_id, + side=side, + price=price, + quantity=quantity, + ) + for op, symbol, order_id, side, price, quantity in rows + ) + + +@dataclass(frozen=True) +class _ActiveOrder: + symbol: str + order_id: int + side: str + price: str + quantity: str + + +def _active_orders(state: BookReplayState) -> Tuple[_ActiveOrder, ...]: + active = [] + for book in state.books: + for side, orders in (("BUY", book.bids), ("SELL", book.asks)): + for order in orders: + active.append( + _ActiveOrder( + book.symbol, + order.order_id, + side, + order.price, + order.quantity, + ) + ) + return tuple(sorted(active, key=lambda item: (item.symbol, item.order_id))) + + +def _new_event( + rng: _SplitMix64, + next_ids: Dict[str, int], +) -> BookReplayEvent: + symbol = rng.choice(tuple(next_ids)) + order_id = next_ids[symbol] + next_ids[symbol] += 1 + return BookReplayEvent( + "add", + symbol, + order_id, + rng.choice(("BUY", "SELL")), + rng.choice(("98", "99", "100", "101", "102")), + rng.choice(("0.5", "1", "2", "3", "5")), + ) + + +def _generated_event( + rng: _SplitMix64, + next_ids: Dict[str, int], + active: Tuple[_ActiveOrder, ...], +) -> BookReplayEvent: + if not active: + return _new_event(rng, next_ids) + roll = rng.randbelow(100) + selected = rng.choice(active) + if roll < 35: + return _new_event(rng, next_ids) + if roll < 60: + mode = rng.randbelow(4) + side = selected.side if mode < 3 else ("SELL" if selected.side == "BUY" else "BUY") + price = selected.price if mode in {0, 1} else rng.choice(("98", "99", "100", "101", "102")) + quantity = ( + str(Decimal(selected.quantity) + Decimal("1")) + if mode == 0 + else rng.choice(("0.5", "1", "2", "3", "5")) + ) + return BookReplayEvent( + "update", + selected.symbol, + selected.order_id, + side, + price, + quantity, + ) + if roll < 74: + return BookReplayEvent("delete", selected.symbol, selected.order_id) + if roll < 91: + return BookReplayEvent( + "probe", + rng.choice(tuple(next_ids)), + side=rng.choice(("BUY", "SELL")), + price=rng.choice(("98", "99", "100", "101", "102")), + quantity=rng.choice(("0.5", "1", "2", "4", "8")), + ) + if roll < 95: + return BookReplayEvent( + "add", + selected.symbol, + selected.order_id, + selected.side, + selected.price, + selected.quantity, + ) + symbol = rng.choice(tuple(next_ids)) + unknown_id = 1_000_000 + next_ids[symbol] + rng.randbelow(10_000) + if roll < 98: + return BookReplayEvent("delete", symbol, unknown_id) + return BookReplayEvent( + "update", + symbol, + unknown_id, + rng.choice(("BUY", "SELL")), + rng.choice(("98", "99", "100", "101", "102")), + rng.choice(("1", "2", "3")), + ) + + +def generate_book_replay_trace( + seed: int, + event_count: int, +) -> Tuple[BookReplayEvent, ...]: + """Generate one stateful trace without consulting candidate behavior.""" + seed = _campaign_seed(seed) + event_count = _positive_int(event_count, "event_count") + rng = _SplitMix64(seed) + scaffold = _semantic_scaffold() + next_ids = {"ALPHA": 1, "BETA": 1} + reference = ReferenceBookReplayAdapter(BookReplayConfig()) + events: List[BookReplayEvent] = [] + active: Tuple[_ActiveOrder, ...] = () + try: + for index in range(1, event_count + 1): + event = scaffold[index - 1] if index <= len(scaffold) else None + if event is None: + event = _generated_event(rng, next_ids, active) + events.append(event) + reference.apply(event, index) + active = _active_orders(reference.snapshot()) + finally: + reference.close() + return tuple(events) + + +@dataclass(frozen=True) +class BookReplayCoverage: + """Source-event capability coverage reached by compared campaign prefixes.""" + + covered: Tuple[str, ...] + + @property + def uncovered(self) -> Tuple[str, ...]: + return tuple(name for name in BOOK_REPLAY_CAPABILITIES if name not in self.covered) + + @property + def complete(self) -> bool: + return not self.uncovered + + def to_dict(self) -> dict: + return { + "covered": len(self.covered), + "expected": len(BOOK_REPLAY_CAPABILITIES), + "capabilities": list(self.covered), + "uncovered": list(self.uncovered), + "complete": self.complete, + } + + +def measure_book_replay_coverage( + traces: Sequence[Sequence[BookReplayEvent]], +) -> BookReplayCoverage: + """Measure which v1 operations occurred before comparison stopped.""" + covered = set() + symbols = set() + for trace in traces: + active: Dict[Tuple[str, int], BookReplayEvent] = {} + for event in trace: + symbols.add(event.symbol) + key = (event.symbol, event.order_id or 0) + if event.op == "add": + covered.add("l3-add") + if key in active: + covered.add("duplicate-active-order-id") + continue + peers = [ + order + for order in active.values() + if order.symbol == event.symbol and order.side == event.side + ] + if any(order.price == event.price for order in peers): + covered.add("same-level-fifo") + if peers and any(order.price != event.price for order in peers): + covered.add("multi-level-depth") + active[key] = event + elif event.op == "update": + previous = active.get(key) + if previous is None: + covered.add("inactive-lifecycle-request") + continue + if previous.side == event.side and previous.price == event.price: + if Decimal(event.quantity or "0") > Decimal(previous.quantity or "0"): + covered.add("same-level-upsize") + else: + covered.add("price-relocation") + active[key] = event + elif event.op == "delete": + if key in active: + covered.add("l3-delete") + del active[key] + else: + covered.add("inactive-lifecycle-request") + elif event.side == "BUY": + covered.add("buy-fill-probe") + else: + covered.add("sell-fill-probe") + if len(symbols) > 1: + covered.add("multiple-symbols") + return BookReplayCoverage(tuple(name for name in BOOK_REPLAY_CAPABILITIES if name in covered)) + + +@dataclass(frozen=True) +class BookReplayCampaignTrace: + index: int + seed: int + events: Tuple[BookReplayEvent, ...] + report: BookReplayReport + + def to_dict(self) -> dict: + return { + "index": self.index, + "seed": self.seed, + "event_count": len(self.events), + "trace_sha256": trace_sha256(self.events), + "compared_events": self.report.compared_events, + "conformant": self.report.conformant, + "divergence": self.report.divergence.to_dict() if self.report.divergence else None, + } + + +@dataclass(frozen=True) +class BookReplayCampaignFailure: + trace: BookReplayCampaignTrace + original_events: Tuple[BookReplayEvent, ...] + original_report: BookReplayReport + minimization: BookReplayMinimizationResult + + def to_dict(self) -> dict: + divergence = self.original_report.divergence + return { + "trace_index": self.trace.index, + "failure_class": ( + f"{divergence.category}:{divergence.kind}" if divergence else "unknown" + ), + "original_divergence_event": divergence.event_index if divergence else None, + "original_event_count": len(self.original_events), + "original_trace_sha256": trace_sha256(self.original_events), + "original_events": [event.to_dict() for event in self.original_events], + "original_report": self.original_report.to_dict(), + "minimized_event_count": len(self.minimization.events), + "minimized_trace_sha256": trace_sha256(self.minimization.events), + "minimized_events": [event.to_dict() for event in self.minimization.events], + "minimization": self.minimization.to_dict(), + "one_minimal": self.minimization.one_minimal, + "budget_exhausted": self.minimization.budget_exhausted, + } + + +@dataclass(frozen=True) +class BookReplayCampaignResult: + seed: int + requested_traces: int + events_per_trace: int + max_minimize_runs: int + candidate_engine: EngineMetadata + traces: Tuple[BookReplayCampaignTrace, ...] + failure: Optional[BookReplayCampaignFailure] + coverage: BookReplayCoverage + + @property + def conformant(self) -> bool: + return self.failure is None + + @property + def campaign_id(self) -> str: + payload = { + "generator_version": BOOK_REPLAY_GENERATOR_VERSION, + "profile": PROFILE_NAME, + "seed": self.seed, + "requested_traces": self.requested_traces, + "events_per_trace": self.events_per_trace, + } + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + @property + def failure_id(self) -> Optional[str]: + if self.failure is None: + return None + divergence = self.failure.minimization.report.divergence + if divergence is None: + raise BookReplayError("campaign failure has no reduced divergence") + payload = { + "campaign_id": self.campaign_id, + "trace_index": self.failure.trace.index, + "original_trace_sha256": trace_sha256(self.failure.original_events), + "reduced_trace_sha256": trace_sha256(self.failure.minimization.events), + "reduced_divergence": divergence.to_dict(), + } + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return "failure-" + hashlib.sha256(encoded).hexdigest()[:20] + + def to_dict(self) -> dict: + minimization_runs = self.failure.minimization.runs if self.failure else 0 + failure = self.failure.to_dict() if self.failure else None + if failure is not None: + failure["failure_id"] = self.failure_id + return { + "schema_version": ARTIFACT_SCHEMA_VERSION, + "artifact_type": "tracebook.book-replay.campaign", + "generator_version": BOOK_REPLAY_GENERATOR_VERSION, + "campaign_id": self.campaign_id, + "profile": PROFILE_NAME, + "config": BookReplayConfig().to_dict(), + "seed": self.seed, + "requested_traces": self.requested_traces, + "completed_traces": len(self.traces), + "events_per_trace": self.events_per_trace, + "generated_events": sum(len(trace.events) for trace in self.traces), + "max_minimize_runs": self.max_minimize_runs, + "candidate_runs": len(self.traces) + minimization_runs, + "candidate_engine": self.candidate_engine.to_dict(), + "stopped_at_first_divergence": True, + "conformant": self.conformant, + "semantic_coverage": self.coverage.to_dict(), + "traces": [trace.to_dict() for trace in self.traces], + "failure": failure, + } + + +def run_book_replay_campaign( + candidate_factory: BookReplayAdapterFactory, + seed: int = 1337, + traces: int = 25, + events_per_trace: int = 100, + max_minimize_runs: int = 100, +) -> BookReplayCampaignResult: + """Run generated traces until completion or the first minimized divergence.""" + seed = _campaign_seed(seed) + traces = _positive_int(traces, "traces") + events_per_trace = _positive_int(events_per_trace, "events_per_trace") + max_minimize_runs = _positive_int(max_minimize_runs, "max_minimize_runs") + config = BookReplayConfig() + results = [] + candidate_engine: Optional[EngineMetadata] = None + failure: Optional[BookReplayCampaignFailure] = None + for trace_index in range(1, traces + 1): + generated_seed = _trace_seed(seed, trace_index) + events = generate_book_replay_trace(generated_seed, events_per_trace) + trace_name = f"campaign:{PROFILE_NAME}:{trace_index}" + report = run_book_replay(events, candidate_factory, config, trace_name=trace_name) + if candidate_engine is None: + candidate_engine = report.candidate_engine + elif candidate_engine != report.candidate_engine: + raise BookReplayError("candidate engine metadata changed between campaign traces") + trace_result = BookReplayCampaignTrace(trace_index, generated_seed, events, report) + results.append(trace_result) + if not report.conformant: + if report.divergence is None: + raise BookReplayError("non-conformant campaign report has no divergence") + original_events = events[: report.divergence.event_index] + original_report = replace( + report, + trace_hash=trace_sha256(original_events), + event_count=len(original_events), + ) + minimization = minimize_book_replay_failure( + original_events, + candidate_factory, + config=config, + max_runs=max_minimize_runs, + trace_name=trace_name, + expected_candidate_engine=candidate_engine, + ) + failure = BookReplayCampaignFailure( + trace_result, + original_events, + original_report, + minimization, + ) + break + if candidate_engine is None: + raise BookReplayError("campaign completed without candidate metadata") + compared = tuple(trace.events[: trace.report.compared_events] for trace in results) + return BookReplayCampaignResult( + seed, + traces, + events_per_trace, + max_minimize_runs, + candidate_engine, + tuple(results), + failure, + measure_book_replay_coverage(compared), + ) + + +__all__ = [ + "BOOK_REPLAY_CAPABILITIES", + "BOOK_REPLAY_GENERATOR_VERSION", + "BookReplayCampaignFailure", + "BookReplayCampaignResult", + "BookReplayCampaignTrace", + "BookReplayCoverage", + "generate_book_replay_trace", + "measure_book_replay_coverage", + "run_book_replay_campaign", +] diff --git a/src/tracebook/book_replay/cli.py b/src/tracebook/book_replay/cli.py index 7b1446a..ce89978 100644 --- a/src/tracebook/book_replay/cli.py +++ b/src/tracebook/book_replay/cli.py @@ -11,11 +11,24 @@ from typing import List, Optional from .._version import __version__ +from .campaign import run_book_replay_campaign from .compare import run_book_replay from .external import ExternalBookReplayAdapterFactory +from .minimize import minimize_book_replay_failure from .model import PROFILE_NAME, BookReplayConfig, BookReplayError, load_book_replay_events +def _add_candidate_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--timeout", type=float, default=5.0) + candidate = parser.add_mutually_exclusive_group(required=True) + candidate.add_argument("--candidate-cmd", help="Candidate command as one shell-style string.") + candidate.add_argument( + "--candidate", + nargs=argparse.REMAINDER, + help="Adapter command and arguments; this must be the final CLI option.", + ) + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m tracebook.book_replay", @@ -34,14 +47,30 @@ def _build_parser() -> argparse.ArgumentParser: run.add_argument("events") run.add_argument("--output") run.add_argument("--profile", choices=[PROFILE_NAME], default=PROFILE_NAME) - run.add_argument("--timeout", type=float, default=5.0) - candidate = run.add_mutually_exclusive_group(required=True) - candidate.add_argument("--candidate-cmd", help="Candidate command as one shell-style string.") - candidate.add_argument( - "--candidate", - nargs=argparse.REMAINDER, - help="Adapter command and arguments; this must be the final CLI option.", + _add_candidate_arguments(run) + + minimize = commands.add_parser( + "minimize", + help="Reduce one divergent L3 trace while preserving its failure category.", ) + minimize.add_argument("events") + minimize.add_argument("--events-output", required=True) + minimize.add_argument("--output") + minimize.add_argument("--max-runs", type=int, default=100) + minimize.add_argument("--profile", choices=[PROFILE_NAME], default=PROFILE_NAME) + _add_candidate_arguments(minimize) + + campaign = commands.add_parser( + "campaign", + help="Generate deterministic L3 traces and minimize the first divergence.", + ) + campaign.add_argument("--output", required=True) + campaign.add_argument("--reduced-events-output") + campaign.add_argument("--seed", type=int, default=1337) + campaign.add_argument("--traces", type=int, default=25) + campaign.add_argument("--events-per-trace", type=int, default=100) + campaign.add_argument("--max-minimize-runs", type=int, default=100) + _add_candidate_arguments(campaign) return parser @@ -80,6 +109,28 @@ def _emit(payload: dict, output: Optional[str]) -> None: print(f"Report written: {path}") +def _write_events(events, output: str) -> None: + path = Path(output) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for event in events: + handle.write( + json.dumps( + event.to_dict(), + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ) + + +def _require_distinct_paths(*paths: Optional[str]) -> None: + resolved = [Path(path).expanduser().resolve() for path in paths if path is not None] + if len(resolved) != len(set(resolved)): + raise BookReplayError("input and output paths must be distinct") + + def main(argv: Optional[List[str]] = None) -> int: args = _build_parser().parse_args(argv) try: @@ -89,9 +140,38 @@ def main(argv: Optional[List[str]] = None) -> int: print(f"Profile: {PROFILE_NAME}") return 0 + if args.command == "campaign": + _require_distinct_paths(args.output, args.reduced_events_output) + campaign = run_book_replay_campaign( + _candidate_factory(args), + seed=args.seed, + traces=args.traces, + events_per_trace=args.events_per_trace, + max_minimize_runs=args.max_minimize_runs, + ) + _emit(campaign.to_dict(), args.output) + if campaign.failure is not None and args.reduced_events_output is not None: + _write_events(campaign.failure.minimization.events, args.reduced_events_output) + print(f"Reduced events written: {Path(args.reduced_events_output)}") + if campaign.failure is None: + return 0 + return 2 if campaign.failure.minimization.report.operational_failure else 1 + events_path = Path(args.events).expanduser().resolve() - if args.output is not None and Path(args.output).expanduser().resolve() == events_path: - raise BookReplayError("input and output paths must be distinct") + events_output = args.events_output if args.command == "minimize" else None + _require_distinct_paths(str(events_path), args.output, events_output) + if args.command == "minimize": + minimization = minimize_book_replay_failure( + load_book_replay_events(str(events_path)), + _candidate_factory(args), + config=BookReplayConfig(args.profile), + max_runs=args.max_runs, + trace_name=str(events_path), + ) + _write_events(minimization.events, args.events_output) + _emit(minimization.to_dict(), args.output) + print(f"Minimized events written: {Path(args.events_output)}") + return 2 if minimization.report.operational_failure else 1 report = run_book_replay( load_book_replay_events(str(events_path)), _candidate_factory(args), diff --git a/src/tracebook/book_replay/minimize.py b/src/tracebook/book_replay/minimize.py new file mode 100644 index 0000000..377c19e --- /dev/null +++ b/src/tracebook/book_replay/minimize.py @@ -0,0 +1,172 @@ +"""Deterministically reduce an L3 book-replay divergence.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, replace +from numbers import Integral +from typing import Iterable, Optional, Tuple + +from .compare import BookReplayReport, run_book_replay +from .model import ( + ARTIFACT_SCHEMA_VERSION, + BookReplayConfig, + BookReplayError, + BookReplayEvent, + EngineMetadata, + trace_sha256, +) +from .protocol import BookReplayAdapterFactory + + +@dataclass(frozen=True) +class BookReplayMinimizationResult: + """A reduced trace and the report proving it still diverges.""" + + events: Tuple[BookReplayEvent, ...] + original_event_count: int + runs: int + target_category: str + one_minimal: bool + budget_exhausted: bool + report: BookReplayReport + + def to_dict(self) -> dict: + minimized_count = len(self.events) + reduction_percent = ( + 100.0 * (self.original_event_count - minimized_count) / self.original_event_count + if self.original_event_count + else 0.0 + ) + return { + "schema_version": ARTIFACT_SCHEMA_VERSION, + "artifact_type": "tracebook.book-replay.minimization", + "original_event_count": self.original_event_count, + "minimized_event_count": minimized_count, + "reduction_percent": reduction_percent, + "runs": self.runs, + "target_category": self.target_category, + "one_minimal": self.one_minimal, + "budget_exhausted": self.budget_exhausted, + "minimized_trace_sha256": trace_sha256(self.events), + "book_replay_report": self.report.to_dict(), + } + + +def _reject_operational_failure( + report: BookReplayReport, + target_category: str, + run_number: int, +) -> None: + divergence = report.divergence + if target_category != "protocol" and divergence is not None and report.operational_failure: + kind = "adapter_close_error" if divergence.close_error is not None else divergence.kind + message = divergence.close_error or divergence.message + raise BookReplayError( + "candidate protocol failure during semantic minimization " + f"run {run_number}: {kind}: {message}" + ) + + +def minimize_book_replay_failure( + events: Iterable[BookReplayEvent], + candidate_factory: BookReplayAdapterFactory, + config: Optional[BookReplayConfig] = None, + max_runs: int = 100, + trace_name: Optional[str] = None, + expected_candidate_engine: Optional[EngineMetadata] = None, +) -> BookReplayMinimizationResult: + """Delta-debug a failure while preserving its first-divergence category.""" + if isinstance(max_runs, bool) or not isinstance(max_runs, Integral) or max_runs <= 0: + raise BookReplayError("max_runs must be a positive integer") + max_runs = int(max_runs) + config = config or BookReplayConfig() + original = tuple(events) + initial = run_book_replay(original, candidate_factory, config, trace_name=trace_name) + runs = 1 + if initial.conformant or initial.divergence is None: + raise BookReplayError("trace is conformant and cannot be minimized") + candidate_engine = expected_candidate_engine or initial.candidate_engine + if initial.candidate_engine != candidate_engine: + raise BookReplayError("candidate engine metadata changed during minimization") + + target = initial.divergence.category + _reject_operational_failure(initial, target, runs) + prefix_length = initial.divergence.event_index + current = original[:prefix_length] if prefix_length > 0 else original + current_report = ( + replace( + initial, + trace_name=None, + trace_hash=trace_sha256(current), + event_count=len(current), + ) + if current != original + else initial + ) + granularity = 2 + one_minimal = len(current) == 0 + + while len(current) >= 2 and runs < max_runs: + chunk_size = int(math.ceil(len(current) / granularity)) + reduced = False + completed_round = True + for start in range(0, len(current), chunk_size): + if runs >= max_runs: + completed_round = False + break + trial = current[:start] + current[start + chunk_size :] + if not trial: + continue + report = run_book_replay(trial, candidate_factory, config) + runs += 1 + if report.candidate_engine != candidate_engine: + raise BookReplayError("candidate engine metadata changed during minimization") + _reject_operational_failure(report, target, runs) + if ( + not report.conformant + and report.divergence is not None + and report.divergence.category == target + ): + current = trial + current_report = report + granularity = max(2, granularity - 1) + reduced = True + break + if reduced: + one_minimal = len(current) == 0 + continue + if not completed_round: + break + if granularity >= len(current): + one_minimal = True + break + granularity = min(len(current), granularity * 2) + + if len(current) == 1 and not one_minimal and runs < max_runs: + empty_report = run_book_replay((), candidate_factory, config) + runs += 1 + if empty_report.candidate_engine != candidate_engine: + raise BookReplayError("candidate engine metadata changed during minimization") + _reject_operational_failure(empty_report, target, runs) + if ( + not empty_report.conformant + and empty_report.divergence is not None + and empty_report.divergence.category == target + ): + current = () + current_report = empty_report + one_minimal = True + + return BookReplayMinimizationResult( + events=current, + original_event_count=len(original), + runs=runs, + target_category=target, + one_minimal=one_minimal, + budget_exhausted=runs >= max_runs and not one_minimal, + report=current_report, + ) + + +__all__ = ["BookReplayMinimizationResult", "minimize_book_replay_failure"] diff --git a/tests/fixtures/faulty_book_replay_adapter.py b/tests/fixtures/faulty_book_replay_adapter.py new file mode 100644 index 0000000..4adc250 --- /dev/null +++ b/tests/fixtures/faulty_book_replay_adapter.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Protocol fixture that visibly requeues one same-level size increase.""" + +from tracebook.book_replay import ( + BookReplayObservation, + BookReplaySnapshot, + BookReplayState, + EngineMetadata, + ReferenceBookReplayAdapter, + serve_book_replay_stdio, +) + + +class FaultyBookReplayAdapter: + def __init__(self, config): + self._inner = ReferenceBookReplayAdapter(config) + self._reorder = False + self.metadata = EngineMetadata( + "faulty-book-replay-test-adapter", + "1", + "Python", + ) + + def apply(self, event, index): + observation = self._inner.apply(event, index) + if event.op == "update" and event.order_id == 1: + self._reorder = True + state = self.snapshot() + return BookReplayObservation( + index, + observation.outcome, + observation.fills, + state.digest(), + state.order_count, + ) + + def snapshot(self): + state = self._inner.snapshot() + if not self._reorder: + return state + books = [] + for book in state.books: + bids = list(book.bids) + if book.symbol == "TEST" and len(bids) >= 2: + bids[0], bids[1] = bids[1], bids[0] + books.append(BookReplaySnapshot(book.symbol, tuple(bids), book.asks)) + return BookReplayState(tuple(books)) + + def close(self): + self._inner.close() + + +if __name__ == "__main__": + raise SystemExit(serve_book_replay_stdio(FaultyBookReplayAdapter)) diff --git a/tests/test_book_replay_campaign.py b/tests/test_book_replay_campaign.py new file mode 100644 index 0000000..d0f4c54 --- /dev/null +++ b/tests/test_book_replay_campaign.py @@ -0,0 +1,259 @@ +import json +import sys +from pathlib import Path + +import pytest + +from tracebook.book_replay import ( + BOOK_REPLAY_CAPABILITIES, + BookReplayError, + BookReplayEvent, + BookReplayObservation, + BookReplaySnapshot, + BookReplayState, + EngineMetadata, + ReferenceBookReplayAdapter, + generate_book_replay_trace, + measure_book_replay_coverage, + minimize_book_replay_failure, + run_book_replay_campaign, +) +from tracebook.book_replay.cli import main +from tracebook.book_replay.model import trace_sha256 + +ROOT = Path(__file__).parents[1] +FAULTY_ADAPTER = ROOT / "tests" / "fixtures" / "faulty_book_replay_adapter.py" + + +class _ReorderingAdapter: + def __init__(self, config): + self._inner = ReferenceBookReplayAdapter(config) + self._reorder = False + self.metadata = EngineMetadata("reordering-test-adapter", "1", "Python") + + def apply(self, event, index): + observation = self._inner.apply(event, index) + if event.op == "update" and event.order_id == 1: + self._reorder = True + state = self.snapshot() + return BookReplayObservation( + index, + observation.outcome, + observation.fills, + state.digest(), + state.order_count, + ) + + def snapshot(self): + state = self._inner.snapshot() + if not self._reorder: + return state + books = [] + for book in state.books: + bids = list(book.bids) + if book.symbol == "TEST" and len(bids) >= 2: + bids[0], bids[1] = bids[1], bids[0] + books.append(BookReplaySnapshot(book.symbol, tuple(bids), book.asks)) + return BookReplayState(tuple(books)) + + def close(self): + self._inner.close() + + +def test_generator_is_deterministic_and_scaffold_covers_every_capability(): + first = generate_book_replay_trace(123, 40) + repeated = generate_book_replay_trace(123, 40) + different = generate_book_replay_trace(124, 40) + + assert first == repeated + assert first != different + assert len(first) == 40 + assert trace_sha256(first) == ( + "4199853cf0496a08beb964283932a6880bd23e534a5e1f72952a4d2d616d0fad" + ) + assert trace_sha256(first[:18]) == ( + "25180672acc45a7487cf3f87abcdf9db5426bdefa82a0d2a00efb259581e41ee" + ) + coverage = measure_book_replay_coverage([first[:18]]) + assert coverage.complete is True + assert coverage.covered == BOOK_REPLAY_CAPABILITIES + + +def test_reference_campaign_is_reproducible_and_complete(): + result = run_book_replay_campaign( + ReferenceBookReplayAdapter, + seed=20260831, + traces=3, + events_per_trace=60, + ) + + assert result.conformant is True + assert result.campaign_id == ( + "sha256:ae989f8ddd45385471c3e6b4b7736ec89410947bd028aadfc1c46f9e294aa876" + ) + assert result.coverage.complete is True + assert result.failure_id is None + assert [trace.seed for trace in result.traces] == [ + 10864503699890266417, + 10644828502866079881, + 3445208458078199137, + ] + + +def test_minimizer_produces_one_minimal_priority_failure(): + events = generate_book_replay_trace(123, 40) + + result = minimize_book_replay_failure(events, _ReorderingAdapter) + + assert [event.to_dict() for event in result.events] == [ + { + "op": "add", + "symbol": "TEST", + "order_id": 1, + "side": "BUY", + "price": "100", + "quantity": "2", + }, + { + "op": "add", + "symbol": "TEST", + "order_id": 2, + "side": "BUY", + "price": "100", + "quantity": "3", + }, + { + "op": "update", + "symbol": "TEST", + "order_id": 1, + "side": "BUY", + "price": "100", + "quantity": "4", + }, + ] + assert result.target_category == "book_state" + assert result.one_minimal is True + assert result.budget_exhausted is False + assert result.report.divergence is not None + assert result.report.divergence.event_index == 3 + + +def test_minimizer_reports_an_exhausted_budget_without_claiming_minimality(): + result = minimize_book_replay_failure( + generate_book_replay_trace(123, 18), + _ReorderingAdapter, + max_runs=1, + ) + + assert result.runs == 1 + assert result.one_minimal is False + assert result.budget_exhausted is True + assert len(result.events) == 3 + + +def test_faulty_campaign_stops_and_minimizes_first_divergence(): + result = run_book_replay_campaign( + _ReorderingAdapter, + seed=20260831, + traces=10, + events_per_trace=100, + ) + + assert result.conformant is False + assert result.failure is not None + assert len(result.traces) == 1 + assert result.traces[0].report.compared_events == 3 + assert len(result.failure.minimization.events) == 3 + assert result.failure.minimization.one_minimal is True + assert result.failure_id == "failure-dfe5c23848b63211b655" + assert trace_sha256(result.failure.minimization.events) == ( + "d9eeb771c7c469c0762c67be7dc76a63a09f36d46c1dd2cda1dbb3cbb47c0b29" + ) + assert result.coverage.covered == ( + "l3-add", + "same-level-fifo", + "same-level-upsize", + ) + + +def test_cli_campaign_writes_report_and_reduced_trace(tmp_path): + report_path = tmp_path / "campaign.json" + reduced_path = tmp_path / "reduced.jsonl" + + exit_code = main( + [ + "campaign", + "--output", + str(report_path), + "--reduced-events-output", + str(reduced_path), + "--seed", + "20260831", + "--traces", + "10", + "--events-per-trace", + "100", + "--candidate-cmd", + f"{sys.executable} {FAULTY_ADAPTER}", + ] + ) + + report = json.loads(report_path.read_text(encoding="utf-8")) + reduced = [ + BookReplayEvent.from_mapping(json.loads(line)) + for line in reduced_path.read_text(encoding="utf-8").splitlines() + ] + assert exit_code == 1 + assert report["artifact_type"] == "tracebook.book-replay.campaign" + assert report["conformant"] is False + assert report["failure"]["minimized_event_count"] == 3 + assert len(reduced) == 3 + + +def test_cli_minimize_writes_a_semantic_failure_artifact(tmp_path): + trace_path = tmp_path / "trace.jsonl" + report_path = tmp_path / "minimization.json" + reduced_path = tmp_path / "reduced.jsonl" + trace_path.write_text( + "".join( + json.dumps(event.to_dict(), sort_keys=True) + "\n" + for event in generate_book_replay_trace(123, 18) + ), + encoding="utf-8", + ) + + exit_code = main( + [ + "minimize", + str(trace_path), + "--events-output", + str(reduced_path), + "--output", + str(report_path), + "--candidate-cmd", + f"{sys.executable} {FAULTY_ADAPTER}", + ] + ) + + report = json.loads(report_path.read_text(encoding="utf-8")) + assert exit_code == 1 + assert report["artifact_type"] == "tracebook.book-replay.minimization" + assert report["target_category"] == "book_state" + assert report["minimized_event_count"] == 3 + assert report["one_minimal"] is True + assert len(reduced_path.read_text(encoding="utf-8").splitlines()) == 3 + + +@pytest.mark.parametrize( + ("keyword", "value", "message"), + [ + ("seed", -1, "seed must be an integer"), + ("traces", 0, "traces must be a positive integer"), + ("events_per_trace", 0, "events_per_trace must be a positive integer"), + ("max_minimize_runs", 0, "max_minimize_runs must be a positive integer"), + ], +) +def test_campaign_rejects_invalid_bounds(keyword, value, message): + arguments = {keyword: value} + with pytest.raises(BookReplayError, match=message): + run_book_replay_campaign(ReferenceBookReplayAdapter, **arguments) diff --git a/tests/test_external_integrations.py b/tests/test_external_integrations.py index 7a9ceee..206e2e4 100644 --- a/tests/test_external_integrations.py +++ b/tests/test_external_integrations.py @@ -86,6 +86,11 @@ def test_nautilus_book_replay_integration_is_version_and_source_pinned(): assert "does not claim" in readme assert '"nautilus-trader==2.0.0rc3"' in workflow assert "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002" in workflow + assert "sha256:09b1599eaeb474d98617acc7869ace26759ed5ab8350803f06197ef572864bab" in workflow + assert "failure-dfe5c23848b63211b655" in readme + assert (NAUTILUS_INTEGRATION / "faulty_adapter.py").is_file() + assert (NAUTILUS_INTEGRATION / "regressions" / "upsize-requeue-reduced.jsonl").is_file() + assert (NAUTILUS_INTEGRATION / "regressions" / "upsize-requeue-failure.json").is_file() @pytest.mark.skipif( From 58b97dba8e646b4351a49790f8ca075bfc30b455 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Mon, 31 Aug 2026 07:30:55 -0400 Subject: [PATCH 13/19] Add submitted-order Go CLOB qualification --- .github/workflows/intrepid-orderbook.yml | 195 +++++++ MANIFEST.in | 3 + README.md | 1 + docs/architecture.md | 1 + integrations/intrepid_orderbook/README.md | 97 ++++ integrations/intrepid_orderbook/adapter.go | 509 ++++++++++++++++++ .../intrepid_orderbook/adapter_test.go | 143 +++++ integrations/intrepid_orderbook/go.mod | 8 + integrations/intrepid_orderbook/go.sum | 4 + integrations/intrepid_orderbook/main.go | 7 + integrations/intrepid_orderbook/protocol.go | 343 ++++++++++++ .../regressions/fok-rejection-failure.json | 58 ++ .../regressions/fok-rejection-reduced.jsonl | 1 + tests/test_external_integrations.py | 32 ++ 14 files changed, 1402 insertions(+) create mode 100644 .github/workflows/intrepid-orderbook.yml create mode 100644 integrations/intrepid_orderbook/README.md create mode 100644 integrations/intrepid_orderbook/adapter.go create mode 100644 integrations/intrepid_orderbook/adapter_test.go create mode 100644 integrations/intrepid_orderbook/go.mod create mode 100644 integrations/intrepid_orderbook/go.sum create mode 100644 integrations/intrepid_orderbook/main.go create mode 100644 integrations/intrepid_orderbook/protocol.go create mode 100644 integrations/intrepid_orderbook/regressions/fok-rejection-failure.json create mode 100644 integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl diff --git a/.github/workflows/intrepid-orderbook.yml b/.github/workflows/intrepid-orderbook.yml new file mode 100644 index 0000000..ecc15b0 --- /dev/null +++ b/.github/workflows/intrepid-orderbook.yml @@ -0,0 +1,195 @@ +name: intrepidkarthi orderbook integration + +on: + workflow_dispatch: + schedule: + - cron: "17 7 * * 2" + pull_request: + paths: + - ".github/workflows/intrepid-orderbook.yml" + - "integrations/intrepid_orderbook/**" + - "src/tracebook/conformance/**" + - "src/tracebook/events/market_replay.py" + - "tests/test_external_integrations.py" + +permissions: + contents: read + +jobs: + pinned-go-clob: + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + + - uses: actions/setup-go@v6 + with: + go-version: "1.23.5" + cache-dependency-path: integrations/intrepid_orderbook/go.sum + + - name: Configure integration paths + run: | + echo "ADAPTER=$RUNNER_TEMP/tracebook-intrepid-orderbook" >> "$GITHUB_ENV" + echo "REPORT_ROOT=$RUNNER_TEMP/intrepid-orderbook" >> "$GITHUB_ENV" + + - name: Install Tracebook + run: python -m pip install -e . + + - name: Build and test the pinned adapter + working-directory: integrations/intrepid_orderbook + run: | + test -z "$(gofmt -l .)" + go mod verify + test "$(go list -m -f '{{.Version}}' github.com/intrepidkarthi/orderbook)" = "v0.26.0" + go vet ./... + go test ./... + go build -trimpath -o "$ADAPTER" . + + - name: Verify the nine-case suite boundary + run: | + set +e + tracebook-conformance suite \ + src/tracebook/conformance/fixtures/v2 \ + --output "$REPORT_ROOT/suite.json" \ + --timeout 10 \ + --candidate "$ADAPTER" + rc=$? + set -e + test "$rc" -eq 1 + python - <<'PY' + import json + import os + from pathlib import Path + + report = json.loads((Path(os.environ["REPORT_ROOT"]) / "suite.json").read_text()) + assert report["case_count"] == 9 + assert report["conformant_cases"] == 6 + failed = { + case["name"]: case["report"]["divergence"] + for case in report["cases"] + if not case["report"]["conformant"] + } + assert set(failed) == { + "order-instructions", + "pro-rata-allocation", + "stp-cancel-incoming", + } + for divergence in failed.values(): + assert divergence["category"] == "outcome" + assert divergence["path"] == "$.observation.outcome.reason" + assert divergence["candidate"] == "INVALID_ORDER" + assert divergence["reference"] is None + PY + + - name: Qualify fifo-limit-v1 + run: | + tracebook-conformance qualify \ + --profile fifo-limit-v1 \ + --seed 42 \ + --traces 25 \ + --events-per-trace 200 \ + --max-minimize-runs 100 \ + --candidate-cmd "$ADAPTER" \ + --output-dir "$REPORT_ROOT/fifo-limit" + python - <<'PY' + import json + import os + from pathlib import Path + + report = json.loads( + (Path(os.environ["REPORT_ROOT"]) / "fifo-limit" / "qualification.json").read_text() + ) + assert report["qualified"] is True + assert report["qualification_id"] == "sha256:f8db775baabe7e665d45bbb920a67e43d405acf445de4951617df68ffa24eb69" + assert report["campaign"]["campaign_id"] == "sha256:59d70645ff13f12fa4af23af69631714df22ef5f25cd1104b0c1124f98f71f6a" + assert report["candidate_runs"] == 28 + assert report["suite"]["report"]["conformant_cases"] == 3 + assert report["campaign"]["completed_traces"] == 25 + assert report["campaign"]["generated_events"] == 5000 + assert report["campaign"]["semantic_coverage"]["covered_count"] == 10 + assert report["campaign"]["semantic_coverage"]["coverage_ratio"] == 1.0 + PY + + - name: Qualify fifo-partial-fill-v1 + run: | + tracebook-conformance qualify \ + --profile fifo-partial-fill-v1 \ + --seed 42 \ + --traces 25 \ + --events-per-trace 200 \ + --max-minimize-runs 100 \ + --candidate-cmd "$ADAPTER" \ + --output-dir "$REPORT_ROOT/fifo-partial" + python - <<'PY' + import json + import os + from pathlib import Path + + report = json.loads( + (Path(os.environ["REPORT_ROOT"]) / "fifo-partial" / "qualification.json").read_text() + ) + assert report["qualified"] is True + assert report["qualification_id"] == "sha256:08bf641bfa57da1892d744c08bca0be00f5ac1ade580ae8fcf7f195f0fcad6eb" + assert report["campaign"]["campaign_id"] == "sha256:297f895b99ac26f92287ababd1dcc25b68b0be98787e9b9d814d5cd241769cf9" + assert report["candidate_runs"] == 28 + assert report["suite"]["report"]["conformant_cases"] == 3 + assert report["campaign"]["completed_traces"] == 25 + assert report["campaign"]["generated_events"] == 5000 + assert report["campaign"]["semantic_coverage"]["covered_count"] == 10 + assert report["campaign"]["semantic_coverage"]["coverage_ratio"] == 1.0 + PY + + - name: Regenerate the full-profile FOK divergence + run: | + set +e + tracebook-conformance qualify \ + --profile fifo-full-v1 \ + --seed 42 \ + --traces 25 \ + --events-per-trace 200 \ + --max-minimize-runs 100 \ + --candidate-cmd "$ADAPTER" \ + --output-dir "$REPORT_ROOT/fifo-full" + rc=$? + set -e + test "$rc" -eq 1 + cmp \ + integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl \ + "$REPORT_ROOT/fifo-full/reduced.jsonl" + cmp \ + integrations/intrepid_orderbook/regressions/fok-rejection-failure.json \ + "$REPORT_ROOT/fifo-full/failure.json" + python - <<'PY' + import json + import os + from pathlib import Path + + report = json.loads( + (Path(os.environ["REPORT_ROOT"]) / "fifo-full" / "qualification.json").read_text() + ) + failure = report["campaign"]["failure"] + retained = json.loads( + (Path(os.environ["REPORT_ROOT"]) / "fifo-full" / "failure.json").read_text() + ) + assert report["qualified"] is False + assert report["qualification_id"] == "sha256:81a095f9804852ff7035f4c3d428a51ae3d0e8e4b81ec335128c6af15b6f65bd" + assert report["campaign"]["campaign_id"] == "sha256:61464afc70170352bc6eecb6ccb22f120d05166d6f5cb7926ce5d43adf8a6b41" + assert failure["failure_id"] == "failure-1ce2954857800b3a068d" + assert failure["original_event_count"] == 7 + assert failure["minimized_event_count"] == 1 + assert failure["one_minimal"] is True + assert retained["expected_reduced_divergence"]["path"] == "$.observation.outcome.reason" + PY + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: intrepid-orderbook-qualification + path: ${{ runner.temp }}/intrepid-orderbook + if-no-files-found: error diff --git a/MANIFEST.in b/MANIFEST.in index c7606c1..6d1d622 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,6 +24,9 @@ include packaging/tracebook-sim/pyproject.toml include packaging/tracebook-sim/LICENSE recursive-include tools *.py recursive-include integrations *.py *.md *.json *.jsonl +recursive-include integrations/intrepid_orderbook *.go +include integrations/intrepid_orderbook/go.mod +include integrations/intrepid_orderbook/go.sum recursive-include integrations/orderbook_rs/src *.rs include integrations/orderbook_rs/Cargo.toml include integrations/orderbook_rs/Cargo.lock diff --git a/README.md b/README.md index 9b07bee..d3d1394 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,7 @@ the canonical protocol. | Candidate | Native surface | Evidence | | --- | --- | --- | +| [`intrepidkarthi/orderbook` v0.26.0](https://github.com/Taz33m/tracebook/tree/main/integrations/intrepid_orderbook) | Go submitted-order CLOB using native process, cancel, reduce, replace, FIFO/pro-rata, STP, trades, and resting-order inspection | `fifo-limit-v1` and `fifo-partial-fill-v1` each pass 3/3 fixed cases, 25/25 generated traces, 5,000 events, and 10/10 capabilities; one native unfillable-FOK rejection is retained as the explicit `fifo-full-v1` contract difference | | [`orderbook-rs` 0.12.0](https://github.com/Taz33m/tracebook/tree/main/integrations/orderbook_rs) | Rust FIFO lifecycle, market/IOC/FOK, STP, deterministic trade IDs, consumption-order queue snapshots | Conformant generated FIFO campaign; `7/9` standard cases with pro-rata and one STP policy difference explicit; semantics reviewed in upstream [issue #203](https://github.com/joaquinbejar/OrderBook-rs/issues/203) and Tracebook [issue #57](https://github.com/Taz33m/tracebook/issues/57) | | [`gocronx/matcher` 0.2.0](https://github.com/Taz33m/tracebook/tree/main/integrations/gocronx_matcher) | Pinned Rust FIFO limit lifecycle over native events and snapshot format v1 | `fifo-limit-v1` qualification passes 3/3 fixed cases, 25/25 generated traces, and 10/10 capabilities; upstream observation and replacement questions remain explicitly pending in [issue #7](https://github.com/gocronx/matcher/issues/7) | | Historical `orderbook-rs` issue #88 | Exact affected Rust dependency behind an opt-in Cargo feature | Flash sequence `15738` and generated event `173` independently reduce to four-event regressions | diff --git a/docs/architecture.md b/docs/architecture.md index 1233da1..3ae7559 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -81,6 +81,7 @@ flowchart TD | `src/tracebook/conformance/suite.py` | Hash verification, materialization, and execution of standard adversarial cases | | `integrations/orderbook_rs/` | Native Rust protocol server and semantic adapter over pinned `orderbook-rs` | | `integrations/gocronx_matcher/` | Native Rust protocol server and `fifo-limit-v1` adapter over pinned `gocronx/matcher` | +| `integrations/intrepid_orderbook/` | Native Go protocol server over a pinned submitted-order CLOB, with FIFO qualification and an explicit FOK contract difference | | `integrations/rust_protocol/` | Shared Rust protocol framing, validation, canonical state hashing, and server loop | | `src/tracebook/simulation/order_generator.py` | Synthetic order streams and event objects | | `src/tracebook/simulation/simulation_engine.py` | Multi-symbol simulation loop and lifecycle event injection | diff --git a/integrations/intrepid_orderbook/README.md b/integrations/intrepid_orderbook/README.md new file mode 100644 index 0000000..70d1895 --- /dev/null +++ b/integrations/intrepid_orderbook/README.md @@ -0,0 +1,97 @@ +# intrepidkarthi/orderbook CLOB Integration + +This adapter checks a genuine submitted-order versus submitted-order central +limit order book. Unlike Tracebook's NautilusTrader book-replay integration, +both sides of every trade are orders sent through the candidate engine. + +The external input is pinned exactly: + +- Go module: `github.com/intrepidkarthi/orderbook v0.26.0`; +- tag: `v0.26.0`; +- source commit: `51d480cdb68b9989febb0b075d291cf891f425b3`; +- repository: ; and +- upstream license: MIT. + +`go.mod` and `go.sum` lock the module graph. No upstream source or compiled +binary is vendored in Tracebook. + +## What The Adapter Exercises + +There is one native `matching.Engine` per symbol. New limit, market, IOC, and +FOK orders call `Engine.Process`; cancel, reduce, and replace call the native +methods of the same names. Trades come directly from the native +`matching.MatchResult`, and snapshots traverse native `RestingOrders`. +Tracebook source IDs are mapped to engine-generated IDs without changing the +engine's matching decisions. + +The protocol configuration maps FIFO and pro-rata allocation plus `NONE`, +`CANCEL_RESTING`, and `CANCEL_INCOMING` self-trade policies to their native +settings. Prices are converted exactly to integer ticks. Quantities are first +normalized at the protocol precision and then represented as integer native +lots. The adapter accepts at most six non-zero quantity decimal places because +the upstream pro-rata implementation multiplies two `int64` lot values; values +that cannot be represented exactly at that boundary are explicitly rejected. + +The local Go tests include a submitted-order crossing test: two same-price +bids are reduced and then consumed by an incoming sell in native FIFO order. +That guards the central claim that this is a matching-engine integration, not +an order-book mirror or a reimplemented reference engine. + +## Qualification Results + +Using Tracebook 0.6.0, generator version 2, seed `42`, 25 traces of 200 events, +and the pinned candidate: + +| Profile | Fixed | Generated | Events | Coverage | Result | +| --- | ---: | ---: | ---: | ---: | --- | +| `fifo-limit-v1` | 3/3 | 25/25 | 5,000 | 10/10 | pass | +| `fifo-partial-fill-v1` | 3/3 | 25/25 | 5,000 | 10/10 | pass | +| `fifo-full-v1` | 4/5 | stopped at trace 1, event 7 | 7 before stop | n/a | expected divergence | + +The passing proof IDs are: + +- FIFO limit qualification `sha256:f8db775baabe7e665d45bbb920a67e43d405acf445de4951617df68ffa24eb69`, + campaign `sha256:59d70645ff13f12fa4af23af69631714df22ef5f25cd1104b0c1124f98f71f6a`; +- FIFO partial-fill qualification `sha256:08bf641bfa57da1892d744c08bca0be00f5ac1ade580ae8fcf7f195f0fcad6eb`, + campaign `sha256:297f895b99ac26f92287ababd1dcc25b68b0be98787e9b9d814d5cd241769cf9`. + +The nine-case standard suite passes 6/9 cases. All three stopped cases reach +the same exact difference: an unfillable FOK is `applied` with no reason in the +Tracebook reference contract, while the native engine returns a rejected +status and its `ErrFOKCannotFill` reason. The adapter preserves that native +result as `rejected / INVALID_ORDER`; it does not rewrite the result to obtain +a green report. This is a contract difference, not by itself proof of an +upstream defect. + +## Retained Divergence + +The full-profile campaign reduces the FOK difference from seven events to the +single event in +[`regressions/fok-rejection-reduced.jsonl`](regressions/fok-rejection-reduced.jsonl). +The retained +[`regressions/fok-rejection-failure.json`](regressions/fok-rejection-failure.json) +records failure ID `failure-1ce2954857800b3a068d`, the original and reduced +trace hashes, and `$.observation.outcome.reason` as the semantic path. CI +regenerates and byte-compares both artifacts. + +## Reproduce + +From this directory: + +```bash +go test ./... +go build -trimpath -o /tmp/tracebook-intrepid-orderbook . + +tracebook-conformance qualify \ + --profile fifo-limit-v1 \ + --seed 42 \ + --traces 25 \ + --events-per-trace 200 \ + --max-minimize-runs 100 \ + --candidate-cmd /tmp/tracebook-intrepid-orderbook \ + --output-dir /tmp/intrepid-fifo-limit +``` + +The tag is reproducible evidence, not a production-readiness certification. +This integration makes no claim about upstream adoption, operational maturity, +latency, durability, or suitability for live trading. diff --git a/integrations/intrepid_orderbook/adapter.go b/integrations/intrepid_orderbook/adapter.go new file mode 100644 index 0000000..b3cf34d --- /dev/null +++ b/integrations/intrepid_orderbook/adapter.go @@ -0,0 +1,509 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "sort" + "strconv" + "time" + + "github.com/intrepidkarthi/orderbook/pkg/matching" + "github.com/intrepidkarthi/orderbook/pkg/types" + "github.com/shopspring/decimal" +) + +type adapterConfig struct { + matchingAlgorithm string + tickSize decimal.Decimal + quantityPlaces uint32 + nativeQtyPlaces uint32 + quantityScale int64 + stp matching.SelfTradePrevention +} + +func parseConfig(wire configWire) (adapterConfig, error) { + algorithm := wire.MatchingAlgorithm + if algorithm == "" { + algorithm = "fifo" + } + if algorithm != "fifo" && algorithm != "pro_rata" { + return adapterConfig{}, errors.New("matching_algorithm must be 'fifo' or 'pro_rata'") + } + tickText := wire.TickSize + if tickText == "" { + tickText = "0.01" + } + tick, err := decimal.NewFromString(tickText) + if err != nil || !tick.IsPositive() { + return adapterConfig{}, errors.New("tick_size must be a positive finite decimal") + } + places := uint32(12) + if wire.QuantityDecimalPlaces != nil { + places = *wire.QuantityDecimalPlaces + } + if places > 18 { + return adapterConfig{}, errors.New("quantity_decimal_places must be between 0 and 18") + } + // The upstream pro-rata path multiplies two int64 lot counts. Scaling all + // protocol quantities by 10^12 would overflow that native multiplication on + // ordinary five-lot fixtures. Six decimal places preserve the campaign's + // exact values while keeping its bounded native arithmetic safely in range. + nativePlaces := min(places, uint32(6)) + scale := int64(1) + for range nativePlaces { + if scale > math.MaxInt64/10 { + return adapterConfig{}, errors.New("quantity scale exceeds int64") + } + scale *= 10 + } + policy := wire.SelfTradePolicy + if policy == "" { + policy = "NONE" + } + var stp matching.SelfTradePrevention + switch policy { + case "NONE": + stp = matching.STPAllow + case "CANCEL_RESTING": + stp = matching.STPCancelOldest + case "CANCEL_INCOMING": + stp = matching.STPCancelNewest + default: + return adapterConfig{}, errors.New( + "self_trade_policy must be NONE, CANCEL_RESTING, or CANCEL_INCOMING", + ) + } + return adapterConfig{algorithm, tick, places, nativePlaces, scale, stp}, nil +} + +func (config adapterConfig) priceTicks(value *jsonNumber) (int64, error) { + if value == nil { + return 0, errors.New("price is required") + } + price, err := decimal.NewFromString(value.String()) + if err != nil { + return 0, errors.New("price must be finite") + } + ticks := price.Div(config.tickSize).RoundBank(0).BigInt() + if !ticks.IsInt64() || ticks.Sign() <= 0 { + return 0, errors.New("price snaps to a non-positive or unsupported tick") + } + return ticks.Int64(), nil +} + +// jsonNumber is an alias so conversion helpers read cleanly without allowing +// floating-point parsing into this integer-tick boundary. +type jsonNumber = json.Number + +func (config adapterConfig) quantityUnits(value *jsonNumber) (int64, error) { + if value == nil { + return 0, errors.New("quantity is required") + } + quantity, err := decimal.NewFromString(value.String()) + if err != nil || !quantity.IsPositive() { + return 0, errors.New("quantity must be positive") + } + normalized := quantity.RoundBank(int32(config.quantityPlaces)) + scaled := normalized.Mul( + decimal.NewFromInt(config.quantityScale), + ) + if !scaled.Equal(scaled.Truncate(0)) { + return 0, fmt.Errorf( + "quantity requires more than %d native decimal places", + config.nativeQtyPlaces, + ) + } + units := scaled.BigInt() + if !units.IsInt64() || units.Sign() <= 0 { + return 0, errors.New("quantity exceeds the adapter's integer range") + } + return units.Int64(), nil +} + +func (config adapterConfig) formatPrice(ticks int64) string { + return decimal.NewFromInt(ticks).Mul(config.tickSize).String() +} + +func (config adapterConfig) formatQuantity(units int64) string { + return decimal.NewFromInt(units).Div(decimal.NewFromInt(config.quantityScale)).String() +} + +type bookHarness struct { + engine *matching.Engine + sourceToNative map[int64]int64 + nativeToSource map[int64]int64 +} + +func newBook(symbol string, config adapterConfig) *bookHarness { + nativeConfig := matching.DefaultConfig(symbol) + nativeConfig.SelfTradePrevention = config.stp + nativeConfig.ProRata = config.matchingAlgorithm == "pro_rata" + nativeConfig.Clock = func() time.Time { return time.Unix(0, 0).UTC() } + return &bookHarness{ + engine: matching.NewEngine(nativeConfig), + sourceToNative: make(map[int64]int64), + nativeToSource: make(map[int64]int64), + } +} + +func (book *bookHarness) reconcile() { + active := make(map[int64]bool) + for _, order := range book.engine.RestingOrders() { + active[order.ID] = true + } + for nativeID, sourceID := range book.nativeToSource { + if !active[nativeID] { + delete(book.nativeToSource, nativeID) + if book.sourceToNative[sourceID] == nativeID { + delete(book.sourceToNative, sourceID) + } + } + } +} + +type adapter struct { + config adapterConfig + books map[string]*bookHarness +} + +func newAdapter(wire configWire) (*adapter, error) { + config, err := parseConfig(wire) + if err != nil { + return nil, err + } + return &adapter{config: config, books: make(map[string]*bookHarness)}, nil +} + +func (adapter *adapter) book(symbol string) *bookHarness { + book := adapter.books[symbol] + if book == nil { + book = newBook(symbol, adapter.config) + adapter.books[symbol] = book + } + return book +} + +func parseOrderID(event marketEvent) (int64, error) { + if event.OrderID == nil { + return 0, errors.New("order_id is required") + } + value, err := strconv.ParseInt(event.OrderID.String(), 10, 64) + if err != nil || value <= 0 { + return 0, errors.New("order_id must be a positive integer") + } + return value, nil +} + +func eventOwner(event marketEvent) int64 { + if event.Owner == nil { + return -1 + } + return *event.Owner +} + +func nativeSide(value *string) (types.Side, error) { + if value == nil { + return "", errors.New("side is required") + } + switch *value { + case "BUY": + return types.SideBuy, nil + case "SELL": + return types.SideSell, nil + default: + return "", errors.New("side must be BUY or SELL") + } +} + +func sourceID(book *bookHarness, nativeID int64) (int64, error) { + value, ok := book.nativeToSource[nativeID] + if !ok { + return 0, fmt.Errorf("adapter has no source id for native order %d", nativeID) + } + return value, nil +} + +func (adapter *adapter) convertTrades( + book *bookHarness, + symbol string, + trades []*types.Trade, +) ([]tradeFill, error) { + converted := make([]tradeFill, 0, len(trades)) + for _, trade := range trades { + buyID, err := sourceID(book, trade.BuyOrderID) + if err != nil { + return nil, err + } + sellID, err := sourceID(book, trade.SellOrderID) + if err != nil { + return nil, err + } + converted = append(converted, tradeFill{ + Symbol: symbol, BuyOrderID: buyID, SellOrderID: sellID, + Price: adapter.config.formatPrice(trade.Price), + Quantity: adapter.config.formatQuantity(trade.Quantity), + }) + } + return converted, nil +} + +func (adapter *adapter) apply(event marketEvent, index uint64) (observationFrame, error) { + result := applied() + var trades []tradeFill + if event.Op == "clear" { + adapter.books[event.Symbol] = newBook(event.Symbol, adapter.config) + } else if event.Op == "cancel" || event.Op == "reduce" || event.Op == "replace" { + book := adapter.books[event.Symbol] + if book == nil { + result = rejected("ORDER_NOT_ACTIVE", "order is not active") + } else { + var err error + result, trades, err = adapter.applyLifecycle(book, event) + if err != nil { + return observationFrame{}, err + } + } + } else if event.Op == "new" { + book := adapter.book(event.Symbol) + var err error + result, trades, err = adapter.applyNew(book, event) + if err != nil { + return observationFrame{}, err + } + } else { + result = rejected("INVALID_ORDER", "unsupported event operation") + } + + state, err := adapter.snapshot() + if err != nil { + return observationFrame{}, err + } + digest, err := state.digest() + if err != nil { + return observationFrame{}, err + } + if trades == nil { + trades = []tradeFill{} + } + return observationFrame{ + Type: "observation", Index: index, Outcome: result, Trades: trades, + StateHash: digest, RestingOrderCount: state.orderCount(), + }, nil +} + +func (adapter *adapter) applyNew( + book *bookHarness, + event marketEvent, +) (outcome, []tradeFill, error) { + source, err := parseOrderID(event) + if err != nil { + return rejected("INVALID_ORDER", err.Error()), nil, nil + } + if _, exists := book.sourceToNative[source]; exists { + return rejected("DUPLICATE_ORDER_ID", "order is already active"), nil, nil + } + side, err := nativeSide(event.Side) + if err != nil { + return rejected("INVALID_ORDER", err.Error()), nil, nil + } + quantity, err := adapter.config.quantityUnits(event.Quantity) + if err != nil { + return rejected("INVALID_ORDER", err.Error()), nil, nil + } + orderType := event.OrderType + if orderType == "" { + orderType = "LIMIT" + } + price := int64(0) + if orderType != "MARKET" { + price, err = adapter.config.priceTicks(event.Price) + if err != nil { + return rejected("INVALID_ORDER", err.Error()), nil, nil + } + } + var nativeType types.OrderType + var tif types.TimeInForce + switch orderType { + case "LIMIT": + nativeType, tif = types.OrderTypeLimit, types.TIFGoodTillCancel + case "MARKET": + nativeType, tif = types.OrderTypeMarket, types.TIFImmediateOrCancel + case "IOC": + nativeType, tif = types.OrderTypeLimit, types.TIFImmediateOrCancel + case "FOK": + nativeType, tif = types.OrderTypeLimit, types.TIFFillOrKill + default: + return rejected("INVALID_ORDER", "unsupported order_type"), nil, nil + } + owner := strconv.FormatInt(eventOwner(event), 10) + order, err := types.NewOrder(owner, event.Symbol, side, nativeType, price, quantity, tif) + if err != nil { + return rejected("INVALID_ORDER", err.Error()), nil, nil + } + order.ClientOrderID = strconv.FormatInt(source, 10) + match := book.engine.Process(order) + book.nativeToSource[order.ID] = source + converted, err := adapter.convertTrades(book, event.Symbol, match.Trades) + if err != nil { + return outcome{}, nil, err + } + if order.IsActive() && order.RemainingQty > 0 { + book.sourceToNative[source] = order.ID + } + book.reconcile() + if match.Status == types.OrderStatusRejected { + message := "order was rejected" + if match.RejectionReason != nil { + message = match.RejectionReason.Error() + } + return rejected("INVALID_ORDER", message), converted, nil + } + return applied(), converted, nil +} + +func (adapter *adapter) applyLifecycle( + book *bookHarness, + event marketEvent, +) (outcome, []tradeFill, error) { + source, err := parseOrderID(event) + if err != nil { + return rejected("ORDER_NOT_ACTIVE", err.Error()), nil, nil + } + nativeID, exists := book.sourceToNative[source] + if !exists { + return rejected("ORDER_NOT_ACTIVE", "order is not active"), nil, nil + } + orders := book.engine.RestingOrders() + var current *types.Order + for _, order := range orders { + if order.ID == nativeID { + current = order + break + } + } + if current == nil { + book.reconcile() + return rejected("ORDER_NOT_ACTIVE", "order is not active"), nil, nil + } + owner := current.UserID + switch event.Op { + case "cancel": + if _, err := book.engine.Cancel(nativeID, owner); err != nil { + return rejected("ORDER_NOT_ACTIVE", err.Error()), nil, nil + } + book.reconcile() + return applied(), nil, nil + case "reduce": + reduction, err := adapter.config.quantityUnits(event.Quantity) + if err != nil || reduction > current.RemainingQty { + message := "reduction exceeds remaining quantity" + if err != nil { + message = err.Error() + } + return rejected("INVALID_ORDER", message), nil, nil + } + if reduction == current.RemainingQty { + if _, err := book.engine.Cancel(nativeID, owner); err != nil { + return rejected("ORDER_NOT_ACTIVE", err.Error()), nil, nil + } + } else { + newTotal := current.Quantity - reduction + if _, err := book.engine.Reduce(nativeID, newTotal, owner); err != nil { + return rejected("INVALID_ORDER", err.Error()), nil, nil + } + } + book.reconcile() + return applied(), nil, nil + case "replace": + price := current.Price + if event.Price != nil { + price, err = adapter.config.priceTicks(event.Price) + if err != nil { + return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil + } + } + quantity := current.RemainingQty + if event.Quantity != nil { + quantity, err = adapter.config.quantityUnits(event.Quantity) + if err != nil { + return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil + } + } + replacement, err := types.NewOrder( + owner, event.Symbol, current.Side, types.OrderTypeLimit, + price, quantity, types.TIFGoodTillCancel, + ) + if err != nil { + return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil + } + replacement.ClientOrderID = strconv.FormatInt(source, 10) + match, err := book.engine.Replace(nativeID, owner, replacement) + if err != nil { + return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil + } + book.nativeToSource[replacement.ID] = source + converted, err := adapter.convertTrades(book, event.Symbol, match.Trades) + if err != nil { + return outcome{}, nil, err + } + delete(book.sourceToNative, source) + if replacement.IsActive() && replacement.RemainingQty > 0 { + book.sourceToNative[source] = replacement.ID + } + book.reconcile() + if match.Status == types.OrderStatusRejected { + message := "replacement was rejected" + if match.RejectionReason != nil { + message = match.RejectionReason.Error() + } + return rejected("INVALID_REPLACEMENT", message), converted, nil + } + return applied(), converted, nil + default: + return rejected("INVALID_ORDER", "unsupported lifecycle operation"), nil, nil + } +} + +func (adapter *adapter) snapshot() (bookState, error) { + symbols := make([]string, 0, len(adapter.books)) + for symbol := range adapter.books { + symbols = append(symbols, symbol) + } + sort.Strings(symbols) + state := bookState{Books: make([]bookSnapshot, 0, len(symbols))} + for _, symbol := range symbols { + book := adapter.books[symbol] + snapshot := bookSnapshot{Symbol: symbol, Bids: []restingOrder{}, Asks: []restingOrder{}} + for _, native := range book.engine.RestingOrders() { + source, ok := book.nativeToSource[native.ID] + if !ok { + return bookState{}, fmt.Errorf( + "adapter has no source id for resting native order %d", + native.ID, + ) + } + owner, err := strconv.ParseInt(native.UserID, 10, 64) + if err != nil { + return bookState{}, fmt.Errorf( + "adapter has invalid source owner for native order %d: %w", + native.ID, + err, + ) + } + resting := restingOrder{ + OrderID: source, Price: adapter.config.formatPrice(native.Price), + RemainingQuantity: adapter.config.formatQuantity(native.RemainingQty), + Owner: owner, OrderType: "LIMIT", + } + if native.Side == types.SideBuy { + snapshot.Bids = append(snapshot.Bids, resting) + } else { + snapshot.Asks = append(snapshot.Asks, resting) + } + } + state.Books = append(state.Books, snapshot) + } + return state, nil +} diff --git a/integrations/intrepid_orderbook/adapter_test.go b/integrations/intrepid_orderbook/adapter_test.go new file mode 100644 index 0000000..2686ba1 --- /dev/null +++ b/integrations/intrepid_orderbook/adapter_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func number(value string) *json.Number { + number := json.Number(value) + return &number +} + +func textPointer(value string) *string { return &value } + +func int64Pointer(value int64) *int64 { return &value } + +func TestEmptyStateDigestMatchesProtocolCanonicalJSON(t *testing.T) { + digest, err := (bookState{Books: []bookSnapshot{}}).digest() + if err != nil { + t.Fatal(err) + } + const expected = "dd8681e973bdf802edf26260297e68e24cdbd75c782c90bbaaaddd401df51090" + if digest != expected { + t.Fatalf("digest = %q, want %q", digest, expected) + } +} + +func TestNativeCrossingPreservesFIFOAfterReduction(t *testing.T) { + harness, err := newAdapter(configWire{}) + if err != nil { + t.Fatal(err) + } + events := []marketEvent{ + {Op: "new", Symbol: "TEST", OrderID: number("1"), Side: textPointer("BUY"), + OrderType: "LIMIT", Price: number("100"), Quantity: number("5"), Owner: int64Pointer(10)}, + {Op: "new", Symbol: "TEST", OrderID: number("2"), Side: textPointer("BUY"), + OrderType: "LIMIT", Price: number("100"), Quantity: number("3"), Owner: int64Pointer(20)}, + {Op: "reduce", Symbol: "TEST", OrderID: number("1"), Quantity: number("2")}, + {Op: "new", Symbol: "TEST", OrderID: number("3"), Side: textPointer("SELL"), + OrderType: "LIMIT", Price: number("100"), Quantity: number("4"), Owner: int64Pointer(30)}, + } + var observation observationFrame + for index, event := range events { + observation, err = harness.apply(event, uint64(index+1)) + if err != nil { + t.Fatalf("event %d: %v", index+1, err) + } + if observation.Outcome.Status != "applied" { + t.Fatalf("event %d outcome = %#v", index+1, observation.Outcome) + } + } + + if len(observation.Trades) != 2 { + t.Fatalf("trades = %#v, want two native fills", observation.Trades) + } + if observation.Trades[0].BuyOrderID != 1 || observation.Trades[0].Quantity != "3" { + t.Fatalf("first fill = %#v, want reduced order 1 for quantity 3", observation.Trades[0]) + } + if observation.Trades[1].BuyOrderID != 2 || observation.Trades[1].Quantity != "1" { + t.Fatalf("second fill = %#v, want order 2 for quantity 1", observation.Trades[1]) + } + state, err := harness.snapshot() + if err != nil { + t.Fatal(err) + } + if len(state.Books) != 1 || len(state.Books[0].Bids) != 1 { + t.Fatalf("state = %#v, want one residual bid", state) + } + remaining := state.Books[0].Bids[0] + if remaining.OrderID != 2 || remaining.RemainingQuantity != "2" { + t.Fatalf("remaining bid = %#v, want order 2 quantity 2", remaining) + } +} + +func TestUnfillableFOKPreservesNativeRejection(t *testing.T) { + harness, err := newAdapter(configWire{}) + if err != nil { + t.Fatal(err) + } + observation, err := harness.apply(marketEvent{ + Op: "new", Symbol: "TEST", OrderID: number("1"), Side: textPointer("BUY"), + OrderType: "FOK", Price: number("100"), Quantity: number("1"), Owner: int64Pointer(10), + }, 1) + if err != nil { + t.Fatal(err) + } + if observation.Outcome.Status != "rejected" || observation.Outcome.Reason == nil || + *observation.Outcome.Reason != "INVALID_ORDER" { + t.Fatalf("outcome = %#v, want the upstream FOK rejection", observation.Outcome) + } + if observation.RestingOrderCount != 0 || len(observation.Trades) != 0 { + t.Fatalf("observation = %#v, want no book mutation", observation) + } +} + +func TestQuantityBeyondNativeSixPlaceBoundaryIsRejected(t *testing.T) { + places := uint32(12) + harness, err := newAdapter(configWire{QuantityDecimalPlaces: &places}) + if err != nil { + t.Fatal(err) + } + observation, err := harness.apply(marketEvent{ + Op: "new", Symbol: "TEST", OrderID: number("1"), Side: textPointer("BUY"), + OrderType: "LIMIT", Price: number("100"), Quantity: number("0.0000001"), Owner: int64Pointer(10), + }, 1) + if err != nil { + t.Fatal(err) + } + if observation.Outcome.Status != "rejected" || observation.Outcome.Message == nil || + !strings.Contains(*observation.Outcome.Message, "more than 6 native decimal places") { + t.Fatalf("outcome = %#v, want an explicit native-precision rejection", observation.Outcome) + } +} + +func TestServerHandshakeAndEmptyCompletion(t *testing.T) { + input := strings.Join([]string{ + `{"type":"hello","protocol":"tracebook.conformance","protocol_version":1,"config":{"matching_algorithm":"fifo","tick_size":"0.01","self_trade_policy":"NONE","quantity_decimal_places":12}}`, + `{"type":"finish","event_count":0}`, + "", + }, "\n") + var output bytes.Buffer + if status := serve(strings.NewReader(input), &output); status != 0 { + t.Fatalf("status = %d, output = %s", status, output.String()) + } + decoder := json.NewDecoder(&output) + var ready map[string]any + var complete map[string]any + if err := decoder.Decode(&ready); err != nil { + t.Fatal(err) + } + if err := decoder.Decode(&complete); err != nil { + t.Fatal(err) + } + engine := ready["engine"].(map[string]any) + if ready["type"] != "ready" || engine["name"] != engineName || engine["version"] != engineVersion { + t.Fatalf("ready = %#v", ready) + } + if complete["type"] != "complete" || complete["event_count"] != float64(0) { + t.Fatalf("complete = %#v", complete) + } +} diff --git a/integrations/intrepid_orderbook/go.mod b/integrations/intrepid_orderbook/go.mod new file mode 100644 index 0000000..69ceb33 --- /dev/null +++ b/integrations/intrepid_orderbook/go.mod @@ -0,0 +1,8 @@ +module github.com/Taz33m/tracebook/integrations/intrepid_orderbook + +go 1.23.5 + +require ( + github.com/intrepidkarthi/orderbook v0.26.0 + github.com/shopspring/decimal v1.4.0 +) diff --git a/integrations/intrepid_orderbook/go.sum b/integrations/intrepid_orderbook/go.sum new file mode 100644 index 0000000..13bb2d8 --- /dev/null +++ b/integrations/intrepid_orderbook/go.sum @@ -0,0 +1,4 @@ +github.com/intrepidkarthi/orderbook v0.26.0 h1:ePTONeCYlz4WQ/Qz/8YhvQ2OKsBTI/VHvclhgG3eLFI= +github.com/intrepidkarthi/orderbook v0.26.0/go.mod h1:5HkWKqoS3UmnPaL5y3WIL8X/CynK3fIcY/T/ZdVq35s= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= diff --git a/integrations/intrepid_orderbook/main.go b/integrations/intrepid_orderbook/main.go new file mode 100644 index 0000000..3d27ba5 --- /dev/null +++ b/integrations/intrepid_orderbook/main.go @@ -0,0 +1,7 @@ +package main + +import "os" + +func main() { + os.Exit(serve(os.Stdin, os.Stdout)) +} diff --git a/integrations/intrepid_orderbook/protocol.go b/integrations/intrepid_orderbook/protocol.go new file mode 100644 index 0000000..5dba8b9 --- /dev/null +++ b/integrations/intrepid_orderbook/protocol.go @@ -0,0 +1,343 @@ +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "sort" +) + +const ( + protocolName = "tracebook.conformance" + protocolVersion = 1 + engineName = "intrepidkarthi/orderbook CLOB" + engineVersion = "v0.26.0@51d480cdb68b" +) + +type configWire struct { + MatchingAlgorithm string `json:"matching_algorithm"` + TickSize string `json:"tick_size"` + SelfTradePolicy string `json:"self_trade_policy"` + QuantityDecimalPlaces *uint32 `json:"quantity_decimal_places"` +} + +type helloFrame struct { + Type string `json:"type"` + Protocol string `json:"protocol"` + ProtocolVersion int `json:"protocol_version"` + Config configWire `json:"config"` +} + +type marketEvent struct { + Op string `json:"op"` + Symbol string `json:"symbol"` + OrderID *json.Number `json:"order_id"` + Side *string `json:"side"` + OrderType string `json:"order_type"` + Price *json.Number `json:"price"` + Quantity *json.Number `json:"quantity"` + Owner *int64 `json:"owner"` + TimestampNS *uint64 `json:"timestamp_ns"` +} + +type eventFrame struct { + Type string `json:"type"` + Index uint64 `json:"index"` + Event marketEvent `json:"event"` +} + +type outcome struct { + Status string `json:"status"` + Reason *string `json:"reason"` + Message *string `json:"message"` +} + +func applied() outcome { return outcome{Status: "applied"} } + +func rejected(reason, message string) outcome { + return outcome{Status: "rejected", Reason: &reason, Message: &message} +} + +type tradeFill struct { + Symbol string `json:"symbol"` + BuyOrderID int64 `json:"buy_order_id"` + SellOrderID int64 `json:"sell_order_id"` + Price string `json:"price"` + Quantity string `json:"quantity"` +} + +type restingOrder struct { + OrderID int64 `json:"order_id"` + Price string `json:"price"` + RemainingQuantity string `json:"remaining_quantity"` + Owner int64 `json:"owner"` + OrderType string `json:"order_type"` +} + +type bookSnapshot struct { + Symbol string `json:"symbol"` + Bids []restingOrder `json:"bids"` + Asks []restingOrder `json:"asks"` +} + +type bookState struct { + Books []bookSnapshot `json:"books"` +} + +func (state bookState) orderCount() int { + count := 0 + for _, book := range state.Books { + count += len(book.Bids) + len(book.Asks) + } + return count +} + +func (state bookState) digest() (string, error) { + raw, err := json.Marshal(state) + if err != nil { + return "", err + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + return "", err + } + canonical, err := canonicalJSON(value) + if err != nil { + return "", err + } + hash := sha256.Sum256(canonical) + return hex.EncodeToString(hash[:]), nil +} + +type observationFrame struct { + Type string `json:"type"` + Index uint64 `json:"index"` + Outcome outcome `json:"outcome"` + Trades []tradeFill `json:"trades"` + StateHash string `json:"state_hash"` + RestingOrderCount int `json:"resting_order_count"` +} + +func canonicalJSON(value any) ([]byte, error) { + switch typed := value.(type) { + case nil, bool, string, json.Number: + return json.Marshal(typed) + case []any: + var buffer bytes.Buffer + buffer.WriteByte('[') + for index, item := range typed { + if index > 0 { + buffer.WriteByte(',') + } + encoded, err := canonicalJSON(item) + if err != nil { + return nil, err + } + buffer.Write(encoded) + } + buffer.WriteByte(']') + return buffer.Bytes(), nil + case map[string]any: + keys := make([]string, 0, len(typed)) + for key := range typed { + keys = append(keys, key) + } + sort.Strings(keys) + var buffer bytes.Buffer + buffer.WriteByte('{') + for index, key := range keys { + if index > 0 { + buffer.WriteByte(',') + } + encodedKey, _ := json.Marshal(key) + buffer.Write(encodedKey) + buffer.WriteByte(':') + encodedValue, err := canonicalJSON(typed[key]) + if err != nil { + return nil, err + } + buffer.Write(encodedValue) + } + buffer.WriteByte('}') + return buffer.Bytes(), nil + default: + return nil, fmt.Errorf("unsupported canonical JSON value %T", value) + } +} + +type serverError struct { + code string + message string +} + +func (err *serverError) Error() string { return err.message } + +func protocolError(message string) error { + return &serverError{code: "PROTOCOL_ERROR", message: message} +} + +func adapterError(err error) error { + return &serverError{code: "ADAPTER_ERROR", message: err.Error()} +} + +func readObject(decoder *json.Decoder) (map[string]json.RawMessage, error) { + var object map[string]json.RawMessage + if err := decoder.Decode(&object); err != nil { + return nil, err + } + if object == nil { + return nil, errors.New("protocol message must be an object") + } + return object, nil +} + +func frameType(object map[string]json.RawMessage) (string, error) { + var value string + if err := json.Unmarshal(object["type"], &value); err != nil || value == "" { + return "", errors.New("protocol message requires a string type") + } + return value, nil +} + +func decodeObject(object map[string]json.RawMessage, target any) error { + raw, err := json.Marshal(object) + if err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + return decoder.Decode(target) +} + +func writeFrame(writer *bufio.Writer, value any) error { + if err := json.NewEncoder(writer).Encode(value); err != nil { + return err + } + return writer.Flush() +} + +func serve(input io.Reader, output io.Writer) int { + decoder := json.NewDecoder(input) + decoder.UseNumber() + writer := bufio.NewWriter(output) + err := runServer(decoder, writer) + if err == nil { + return 0 + } + var typed *serverError + if !errors.As(err, &typed) { + typed = &serverError{code: "PROTOCOL_ERROR", message: err.Error()} + } + _ = writeFrame(writer, map[string]any{ + "type": "error", "code": typed.code, "message": typed.message, + }) + return 2 +} + +func runServer(decoder *json.Decoder, writer *bufio.Writer) error { + first, err := readObject(decoder) + if err != nil { + return protocolError("expected hello message: " + err.Error()) + } + var hello helloFrame + if err := decodeObject(first, &hello); err != nil { + return protocolError(err.Error()) + } + if hello.Type != "hello" { + return protocolError("first message must be hello") + } + if hello.Protocol != protocolName { + return protocolError(fmt.Sprintf("protocol must be %q", protocolName)) + } + if hello.ProtocolVersion != protocolVersion { + return protocolError(fmt.Sprintf("protocol_version must be %d", protocolVersion)) + } + adapter, err := newAdapter(hello.Config) + if err != nil { + return protocolError(err.Error()) + } + if err := writeFrame(writer, map[string]any{ + "type": "ready", + "protocol": protocolName, + "protocol_version": protocolVersion, + "engine": map[string]string{ + "name": engineName, "version": engineVersion, "language": "Go", + }, + }); err != nil { + return adapterError(err) + } + + var lastIndex uint64 + for { + object, err := readObject(decoder) + if err != nil { + if errors.Is(err, io.EOF) { + return protocolError("protocol ended before finish") + } + return protocolError(err.Error()) + } + kind, err := frameType(object) + if err != nil { + return protocolError(err.Error()) + } + switch kind { + case "event": + var frame eventFrame + if err := decodeObject(object, &frame); err != nil { + return protocolError(err.Error()) + } + if frame.Index == 0 || frame.Index != lastIndex+1 { + return protocolError("event indexes must be contiguous and start at 1") + } + observation, err := adapter.apply(frame.Event, frame.Index) + if err != nil { + return adapterError(err) + } + lastIndex = frame.Index + if err := writeFrame(writer, observation); err != nil { + return adapterError(err) + } + case "snapshot": + var frame struct { + Index uint64 `json:"index"` + } + if err := decodeObject(object, &frame); err != nil { + return protocolError(err.Error()) + } + if frame.Index != lastIndex { + return protocolError("snapshot index does not match the last event") + } + state, err := adapter.snapshot() + if err != nil { + return adapterError(err) + } + if err := writeFrame(writer, map[string]any{ + "type": "snapshot", "index": lastIndex, "state": state, + }); err != nil { + return adapterError(err) + } + case "finish": + var frame struct { + EventCount uint64 `json:"event_count"` + } + if err := decodeObject(object, &frame); err != nil { + return protocolError(err.Error()) + } + if frame.EventCount != lastIndex { + return protocolError("finish event_count does not match the last event") + } + return writeFrame(writer, map[string]any{ + "type": "complete", "event_count": lastIndex, + }) + default: + return protocolError(fmt.Sprintf("unsupported protocol message type: %q", kind)) + } + } +} diff --git a/integrations/intrepid_orderbook/regressions/fok-rejection-failure.json b/integrations/intrepid_orderbook/regressions/fok-rejection-failure.json new file mode 100644 index 0000000..cdc1e1d --- /dev/null +++ b/integrations/intrepid_orderbook/regressions/fok-rejection-failure.json @@ -0,0 +1,58 @@ +{ + "artifact_type": "tracebook.conformance.failure", + "budget_exhausted": false, + "campaign_id": "sha256:61464afc70170352bc6eecb6ccb22f120d05166d6f5cb7926ce5d43adf8a6b41", + "campaign_seed": 42, + "candidate_engine": { + "language": "Go", + "name": "intrepidkarthi/orderbook CLOB", + "version": "v0.26.0@51d480cdb68b" + }, + "config": { + "matching_algorithm": "fifo", + "quantity_decimal_places": 12, + "self_trade_policy": "NONE", + "tick_size": "0.01" + }, + "expected_reduced_divergence": { + "candidate": "INVALID_ORDER", + "category": "outcome", + "event": { + "op": "new", + "order_id": 2, + "order_type": "FOK", + "owner": 4, + "price": 99.99, + "quantity": 1.0, + "side": "SELL", + "symbol": "BETA", + "timestamp_ns": 7 + }, + "event_index": 1, + "kind": "type_mismatch", + "message": "candidate applied/rejected semantics differ from the reference", + "path": "$.observation.outcome.reason", + "reference": null, + "snapshot_error": null + }, + "failure_class": "order-outcome drift", + "failure_id": "failure-1ce2954857800b3a068d", + "generator_version": 2, + "one_minimal": true, + "original_divergence_event": 7, + "original_event_count": 7, + "original_trace_sha256": "sha256:be56a106f547a3ed17988bd8b6e448b91ed18e111678c9ac1dc9bd384fce7f8f", + "paths": { + "campaign": "campaign.json", + "minimization": "failure/minimization.json", + "original": "original.jsonl", + "reduced": "reduced.jsonl" + }, + "profile": "fifo-full-v1", + "reduced_event_count": 1, + "reduced_trace_sha256": "sha256:395c88526cdd81445205298f8234d8b57afcef71534f8fd7312608fc43862706", + "schema_version": 1, + "target_category": "outcome", + "trace_index": 1, + "trace_seed": 2949826092126892291 +} diff --git a/integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl b/integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl new file mode 100644 index 0000000..fcee8c4 --- /dev/null +++ b/integrations/intrepid_orderbook/regressions/fok-rejection-reduced.jsonl @@ -0,0 +1 @@ +{"op":"new","order_id":2,"order_type":"FOK","owner":4,"price":99.99,"quantity":1.0,"side":"SELL","symbol":"BETA","timestamp_ns":7} diff --git a/tests/test_external_integrations.py b/tests/test_external_integrations.py index 206e2e4..124edaf 100644 --- a/tests/test_external_integrations.py +++ b/tests/test_external_integrations.py @@ -40,6 +40,7 @@ RUST_INTEGRATION = ROOT / "integrations" / "orderbook_rs" GOCRONX_INTEGRATION = ROOT / "integrations" / "gocronx_matcher" NAUTILUS_INTEGRATION = ROOT / "integrations" / "nautilus_trader" +INTREPID_INTEGRATION = ROOT / "integrations" / "intrepid_orderbook" RUST_PROTOCOL = ROOT / "integrations" / "rust_protocol" @@ -199,6 +200,34 @@ def test_gocronx_matcher_integration_is_pinned_qualified_and_honest_about_assump assert "actions/upload-artifact@v7" in workflow +def test_intrepid_orderbook_is_pinned_qualified_and_retains_the_fok_boundary(): + go_mod = (INTREPID_INTEGRATION / "go.mod").read_text(encoding="utf-8") + go_sum = (INTREPID_INTEGRATION / "go.sum").read_text(encoding="utf-8") + readme = (INTREPID_INTEGRATION / "README.md").read_text(encoding="utf-8") + workflow = (ROOT / ".github" / "workflows" / "intrepid-orderbook.yml").read_text( + encoding="utf-8" + ) + root_readme = (ROOT / "README.md").read_text(encoding="utf-8") + architecture = (ROOT / "docs" / "architecture.md").read_text(encoding="utf-8") + + assert "github.com/intrepidkarthi/orderbook v0.26.0" in go_mod + assert "github.com/intrepidkarthi/orderbook v0.26.0 h1:" in go_sum + assert "51d480cdb68b9989febb0b075d291cf891f425b3" in readme + assert "submitted-order versus submitted-order" in readme + assert "at most six non-zero quantity decimal places" in readme + assert "6/9" in readme + assert "failure-1ce2954857800b3a068d" in readme + assert "sha256:f8db775baabe7e665d45bbb920a67e43d405acf445de4951617df68ffa24eb69" in workflow + assert "sha256:08bf641bfa57da1892d744c08bca0be00f5ac1ade580ae8fcf7f195f0fcad6eb" in workflow + assert "sha256:81a095f9804852ff7035f4c3d428a51ae3d0e8e4b81ec335128c6af15b6f65bd" in workflow + assert "$.observation.outcome.reason" in workflow + assert "intrepidkarthi/orderbook` v0.26.0" in root_readme + assert "integrations/intrepid_orderbook/" in architecture + assert (INTREPID_INTEGRATION / "adapter_test.go").is_file() + assert (INTREPID_INTEGRATION / "regressions" / "fok-rejection-reduced.jsonl").is_file() + assert (INTREPID_INTEGRATION / "regressions" / "fok-rejection-failure.json").is_file() + + def test_native_adapters_share_one_rust_protocol_contract(): shared_source = (RUST_PROTOCOL / "src" / "lib.rs").read_text(encoding="utf-8") shared_server = (RUST_PROTOCOL / "src" / "server.rs").read_text(encoding="utf-8") @@ -223,6 +252,9 @@ def test_source_manifest_includes_native_integration_files(): assert "recursive-include integrations/orderbook_rs/src *.rs" in manifest assert package_data["tracebook.conformance.fixtures.v2"] == ["*.json", "*.jsonl"] assert "recursive-include integrations *.py *.md *.json *.jsonl" in manifest + assert "recursive-include integrations/intrepid_orderbook *.go" in manifest + assert "include integrations/intrepid_orderbook/go.mod" in manifest + assert "include integrations/intrepid_orderbook/go.sum" in manifest assert "recursive-include src/tracebook/book_replay/fixtures *.jsonl" in manifest assert "include integrations/orderbook_rs/Cargo.lock" in manifest assert "prune integrations/orderbook_rs/target" in manifest From 55e3c4b010c8cc1bc967db59f94f593ffa0f6838 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Fri, 4 Sep 2026 21:08:03 -0400 Subject: [PATCH 14/19] Harden adapters and packaging; reconcile corrected study and onboarding --- .github/workflows/ci.yml | 3 + .github/workflows/intrepid-orderbook.yml | 2 +- .../workflows/nautilus-trader-book-replay.yml | 4 +- .github/workflows/release.yml | 36 +- .gitignore | 1 + CHANGELOG.md | 14 + MANIFEST.in | 20 +- Makefile | 1 + PROJECT_PLAN.md | 23 +- QUICKSTART.md | 3 + README.md | 4 + ...qualification-generalization-v2-results.md | 119 +++--- docs/book-replay-conformance.md | 4 + docs/commands.md | 6 + docs/conformance.md | 5 + docs/independent-onboarding.md | 145 ++++++++ integrations/intrepid_orderbook/README.md | 66 +++- integrations/intrepid_orderbook/adapter.go | 125 +++++-- .../intrepid_orderbook/adapter_test.go | 342 ++++++++++++++++++ integrations/intrepid_orderbook/protocol.go | 21 +- integrations/nautilus_trader/README.md | 7 + src/tracebook/book_replay/external.py | 22 +- src/tracebook/conformance/external.py | 22 +- src/tracebook/visualization/dashboard.py | 2 +- tests/test_book_replay.py | 66 +++- tests/test_conformance_exit_codes.py | 16 +- tests/test_conformance_shutdown.py | 122 +++++++ tests/test_dashboard_demo_construction.py | 20 +- tests/test_distribution_privacy.py | 179 +++++++++ tests/test_external_integrations.py | 10 +- tests/test_release_metadata.py | 12 +- tools/verify_distribution_privacy.py | 123 +++++++ 32 files changed, 1394 insertions(+), 151 deletions(-) create mode 100644 docs/independent-onboarding.md create mode 100644 tests/test_conformance_shutdown.py create mode 100644 tests/test_distribution_privacy.py create mode 100644 tools/verify_distribution_privacy.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72a7d31..9014c42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,9 @@ jobs: - name: Validate both distributions run: python -m twine check dist/conformance/* dist/simulator/* + - name: Reject private paths in both distributions + run: python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/* + - name: Rebuild wheels from sdists and verify artifact agreement if: matrix.python-version == '3.13' run: | diff --git a/.github/workflows/intrepid-orderbook.yml b/.github/workflows/intrepid-orderbook.yml index ecc15b0..b9c9ad7 100644 --- a/.github/workflows/intrepid-orderbook.yml +++ b/.github/workflows/intrepid-orderbook.yml @@ -23,7 +23,7 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.12" cache: pip diff --git a/.github/workflows/nautilus-trader-book-replay.yml b/.github/workflows/nautilus-trader-book-replay.yml index 64b6437..70c0a91 100644 --- a/.github/workflows/nautilus-trader-book-replay.yml +++ b/.github/workflows/nautilus-trader-book-replay.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.14" cache: pip @@ -32,7 +32,7 @@ jobs: - name: Install Tracebook and the pinned optional candidate run: | python -m pip install --upgrade pip - python -m pip install -e . "nautilus-trader==2.0.0rc3" + python -m pip install -e . "nautilus-trader==2.0.0rc3" "pytest>=9.1.1" - name: Verify the isolated L3 book-replay profile run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5668426..ca579af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,6 +74,9 @@ jobs: python -m build --sdist --wheel --outdir dist/conformance . python -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim + - name: Reject private paths in both distributions + run: python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/* + - name: Rebuild and verify wheel byte-for-byte reproducibility run: | python -m build --wheel --outdir reproducibility/conformance . @@ -142,39 +145,6 @@ jobs: - name: Prove conformance wheel without simulation dependencies run: python tools/smoke_conformance_wheel.py dist/conformance/*.whl - - name: Reject private navigation material in sdist - run: | - python - <<'PY' - import pathlib - import tarfile - - forbidden = { - "AGENTS.md", - "CLAUDE.md", - "docs/openwiki.md", - "docs/graphify.md", - } - forbidden_prefixes = ("openwiki/", "graphify-out/", ".local-tools/") - archives = sorted(pathlib.Path("dist").glob("*/*.tar.gz")) - if len(archives) != 2: - raise SystemExit(f"expected two sdists, found: {archives}") - for archive in archives: - with tarfile.open(archive, "r:gz") as source: - packaged = { - "/".join(pathlib.PurePosixPath(name).parts[1:]) - for name in source.getnames() - } - leaked = sorted( - path - for path in packaged - if path in forbidden or path.startswith(forbidden_prefixes) - ) - if leaked: - raise SystemExit( - f"private navigation material in {archive.name}: {leaked}" - ) - PY - - name: Upload conformance distributions uses: actions/upload-artifact@v7 with: diff --git a/.gitignore b/.gitignore index 2adfec9..68b848d 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ wheels/ # Virtual Environment .venv/ +.venv-ci/ venv/ ENV/ env/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ccca77d..d8ac5e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions. ## Unreleased +- Added a separate L3 book-replay contract and pinned NautilusTrader adapter. + This exercises market-data deltas and non-mutating FIFO fill probes, not a + submitted-order CLOB. Both process clients now honor the configured shutdown + timeout and reject forced termination even if the candidate exits zero. +- Added a pinned native Go CLOB adapter for `intrepidkarthi/orderbook`, with + FIFO qualification and a retained unfillable-FOK contract difference. Adapter + boundaries preserve anonymous-order STP behavior, binary64 price snapping, + output-only quantity precision, and fail closed on unsafe native arithmetic. +- Excluded private experiment trees and local agent/navigation metadata from + source distributions, with a synthetic stale-manifest regression and privacy + checks for source archives, wheels, and embedded source-file inventories. +- Published the corrected 24-run agent-qualification aggregate: 1/12 safe passes + with public docs and 5/12 with the frozen skill. The improvement is confined + to boundary judgment; neither arm safely qualified the compatible engine. - Added a captured two-run qualification workflow. Candidate adapters can now report optional revision and snapshot metadata, candidate-facing commands can enforce a task-pinned identity before the first event, and `evidence-init` / diff --git a/MANIFEST.in b/MANIFEST.in index 6d1d622..9c09167 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,13 +10,6 @@ include SECURITY.md include SUPPORT.md recursive-include docs *.md *.png *.svg -exclude docs/openwiki.md -exclude docs/graphify.md -exclude AGENTS.md -exclude CLAUDE.md -prune openwiki -prune graphify-out -prune .local-tools recursive-include examples *.py *.json *.jsonl *.yml recursive-include experiments *.py *.json recursive-include packaging *.md @@ -49,3 +42,16 @@ recursive-include src/tracebook/corpus/fixtures *.json *.jsonl recursive-include src/tracebook/conformance/fixtures *.json *.jsonl recursive-include src/tracebook/book_replay/fixtures *.jsonl include src/tracebook/py.typed + +# Apply exclusions last, including to entries retained in an old SOURCES.txt. +prune experiments/private +prune .agents +prune .codex +prune openwiki +prune graphify-out +prune .local-tools +exclude docs/openwiki.md +exclude docs/graphify.md +exclude AGENTS.md +exclude CLAUDE.md +global-exclude AGENTS.md CLAUDE.md skills-lock.json diff --git a/Makefile b/Makefile index 9249d1a..2c32fc1 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,7 @@ compile: build: $(PYTHON_VENV) -m build --sdist --wheel --outdir dist/conformance . $(PYTHON_VENV) -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim + $(PYTHON_VENV) tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/* $(PYTHON_VENV) -m twine check dist/conformance/* dist/simulator/* $(PYTHON_VENV) tools/verify_sdist_wheel_agreement.py \ --conformance-wheel dist/conformance/*.whl \ diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index c87f837..7b11a3b 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -21,6 +21,7 @@ handler or trading infrastructure. src/tracebook/ core/ matching, lifecycle, snapshots, deterministic replay conformance/ adapters, campaigns, semantic diffing, minimization, standard suite + book_replay/ separate L3 delta/probe contract; never venue-order matching events/ normalized event replay and offline venue adapters corpus/ safe local capture, manifests, golden state, import benchmarks simulation/ synthetic order flow and paced workload execution @@ -31,6 +32,8 @@ integrations/ python_matching_engine/ pinned external adapter and compatibility trace orderbook_rs/ native Rust adapter, faulty example, regression proof gocronx_matcher/ pinned Rust adapter and profile qualification + intrepid_orderbook/ pinned Go CLOB adapter and explicit FOK divergence + nautilus_trader/ pinned L3 book replay, not CLOB qualification ``` ## External Validation @@ -94,6 +97,13 @@ onboarding work. ## Current Product Hypothesis +The corrected, held-out [24-run agent study](docs/agent-qualification-generalization-v2-results.md) +completed on 2026-08-29. The frozen skill raised safe passes from 1/12 to 5/12, +entirely through better profile-boundary decisions; compatible-engine evidence +passed in neither arm (0/6 each). The now-implemented captured two-run workflow +addresses candidate identity and artifact freshness mechanically. Its effect on +independent onboarding remains unmeasured; the study did not test that workflow. + The immediate bottleneck is adoption, not another matching algorithm. An engine author should be able to move from an adapter command to a trustworthy, profile-scoped compatibility artifact in one invocation. Qualification version @@ -134,10 +144,15 @@ experiments rather than presumed roadmap wins. ## Next Milestones -1. Complete one independent public-package onboarding. Start from a concrete, - tested semantic artifact, ask one narrow ownership question, and record - whether an external maintainer runs the tool or retains its output. Do not - count Tracebook-owned scheduled integrations as adoption. +1. Complete one independent onboarding of the captured two-run workflow. Start + from a concrete, tested semantic artifact, record the exact Tracebook source + revision and wheel hash (the workflow is not in the published 0.6.0 package), + ask one narrow ownership question, and measure time to the first verified + pair, human corrections, and whether an external maintainer retains the + output. Tracebook-owned rehearsals and scheduled integrations are not + adoption. The [prepared Go evidence packet](docs/independent-onboarding.md) + supplies a concrete starting point; independent public-package use remains + the release gate below. 2. Keep the raw Flash-to-Tracebook forensic handoff opportunistic. If an untriaged divergence appears, measure localization, minimization, translation into a native regression, and maintainer retention honestly; do not keep diff --git a/QUICKSTART.md b/QUICKSTART.md index d8db6bb..642ad49 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -121,7 +121,10 @@ Use `--algorithm PRO_RATA` to compare the pro-rata path. ## Start The Dashboard +Install the optional dashboard dependencies before starting the interactive UI: + ```bash +python -m pip install "tracebook-sim[dashboard]==0.6.0" tracebook-dashboard --port 8050 --demo-simulation ``` diff --git a/README.md b/README.md index 62ebf80..23a850b 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,10 @@ these are separate version boundaries. `fifo-limit-v1` qualification does not run STP or pro-rata cases, so unsupported features stay explicit without becoming false failures. +The captured evidence commands below are unreleased source features, not part +of the published 0.6.0 wheel. Record the exact source revision and built wheel +hash when evaluating them. + For release evidence, do not rely on one successful workspace. Start a captured pair with `tracebook-conformance evidence-init`, run the canonical qualification once in each generated clean root with the plan's pinned candidate name, diff --git a/docs/agent-qualification-generalization-v2-results.md b/docs/agent-qualification-generalization-v2-results.md index 4287606..0d5b9f9 100644 --- a/docs/agent-qualification-generalization-v2-results.md +++ b/docs/agent-qualification-generalization-v2-results.md @@ -2,26 +2,28 @@ **Protocol:** `agent-qualification-generalization-v2` **Public protocol commit:** `50ac3b63c934ec9f3c18869cc5c68b221b73fdd2` -**Execution completed:** 2026-08-02 -**Aggregate grading completed:** 2026-08-03 -**Validity:** invalidated on 2026-08-25 after execution-harness review; retained -only as a historical audit record pending a corrected 24-run recollection +**Corrected execution completed:** 2026-08-29 +**Blinded aggregate grading completed:** 2026-08-29 +**Validity:** corrected 24-run recollection complete; this result supersedes +the invalidated 2026-08-02 execution -## Historical Result (Invalidated) +The preregistered protocol file remains byte-frozen; its pending-recollection +status line is therefore a historical execution note, not the current result. -The figures below describe the original execution. They must not be used for -inference: the corrected harness restores fixture-mutation validation, Claude -provider-catalog auditing, and the restricted toolchain environment, among -other isolation and evidence-capture fixes. +## Corrected Result All 24 preregistered runs received valid pre-score technical verdicts before -semantic outcomes were inspected. The frozen skill improved the autonomous -safe-pass rate by 50 percentage points over public documentation alone. +semantic outcomes were inspected. Private gold stayed local. Grading used +opaque evidence IDs, and condition/provider labels were revealed only after the +24 verdicts were sealed. + +The frozen skill improved the autonomous safe-pass rate by **33.3 percentage +points** over public documentation alone. | Condition | Safe passes | Runs | Rate | | --- | ---: | ---: | ---: | | Public docs | 1 | 12 | 8.3% | -| Frozen skill | 7 | 12 | 58.3% | +| Frozen skill | 5 | 12 | 41.7% | The result exceeds the preregistered directional-product-signal threshold of +20 percentage points. The skill condition produced zero unsupported @@ -34,69 +36,90 @@ matching behavior. | Agent | Docs | Skill | Delta | | --- | ---: | ---: | ---: | -| Claude | 1/6 (16.7%) | 3/6 (50.0%) | +33.3 pp | -| Codex | 0/6 (0.0%) | 4/6 (66.7%) | +66.7 pp | +| Claude | 1/6 (16.7%) | 2/6 (33.3%) | +16.7 pp | +| Codex | 0/6 (0.0%) | 3/6 (50.0%) | +50.0 pp | ### By held-out case class | Case | Docs | Skill | Delta | | --- | ---: | ---: | ---: | -| Compatible engine | 0/6 (0.0%) | 1/6 (16.7%) | +16.7 pp | -| Profile boundary | 1/6 (16.7%) | 6/6 (100.0%) | +83.3 pp | - -The intervention generalized strongly to boundary recognition: every skill -run correctly refused to manufacture Tracebook qualification evidence for the -system that did not select identified resting venue makers. Compatible-engine -release evidence remained the bottleneck. Only one of six skill runs satisfied -every clean-build, candidate-identity, canonical-bundle, and byte-equality +| Compatible engine | 0/6 (0.0%) | 0/6 (0.0%) | +0.0 pp | +| Profile boundary | 1/6 (16.7%) | 5/6 (83.3%) | +66.7 pp | + +The intervention generalized to boundary recognition: five of six skill runs +correctly refused to manufacture Tracebook qualification evidence for a system +that does not select identified resting venue makers. One additional skill run +selected the right boundary class but failed the mechanically fresh native-pair gate. +Compatible-engine release evidence remained the bottleneck. Several runs +reached the expected behavioral counts twice, but none preserved a pair whose +candidate metadata and qualification identity matched the frozen canonical +gold. Other runs omitted the canonical pair or selected the wrong class. + +## Matched Safe-Pass Timing + +Only `claude__c5-boundary__r1` safely passed in both conditions. The docs run +took 685.14 seconds and the skill run took 655.93 seconds: 29.21 seconds, or +4.3%, faster. No other timing comparison is reported. + ## Interpretation This is a positive directional product result, not a statistically conclusive model comparison. It supports three narrow claims: -1. an explicit qualification skill materially improved safe autonomous - workflow execution in this held-out cohort; -2. the improvement was driven mainly by profile-boundary judgment; and -3. compatible-engine release evidence still required machine-enforced gates - rather than more prose. +1. the frozen skill improved safe autonomous execution in this corrected + held-out cohort; +2. the gain came entirely from profile-boundary judgment; and +3. compatible-engine qualification still requires machine-enforced identity + and artifact gates. -It does not establish broad model superiority, production adoption, product- -market fit, or reliable autonomous qualification across matching engines. The -adaptive v2 cohort must not be pooled with v1 as one untouched confirmatory -study, revised after grading, or rerun to improve the result. +It does not establish broad model superiority, production adoption, +product-market fit, or reliable autonomous qualification across matching +engines. The adaptive v2 cohort must not be pooled with v1 or the invalidated +execution as one untouched confirmatory study. ## Product Consequence -The highest-leverage follow-up is a public evidence workflow that: +The public [captured two-run evidence workflow](conformance.md#captured-two-run-evidence) +now enforces canonical candidate name, revision, and snapshot binding, then +requires two fresh byte-matching qualification bundles with fixed campaign and +qualification identities. It was implemented separately from the frozen study; +these results do not measure that workflow's effect. + +The next gate is independent use: test the consolidated workflow with an +external maintainer and record time to first verified pair, human corrections, +and whether the maintainer retains its output or CI workflow. -- pins candidate name, revision, and source snapshot before adapter readiness; -- creates two separate candidate, adapter, build, cache, and output roots; -- requires two canonical qualification bundles with matching deterministic - identifiers, counts, coverage, and bytes; and -- emits one concise, grading-ready manifest. +## Corrected Harness Binding -That follow-up is implemented separately in -[PR #71](https://github.com/Taz33m/tracebook/pull/71). It is not part of the -frozen intervention and must be evaluated on another untouched candidate. +The measured recollection used: + +- execution plan SHA-256: + `1d3753bf1ea7e46b261e56ebb36fdf726c56192cb19d50c2dd00b6213ca68d1f`; +- execution freeze SHA-256: + `82b037d86723fc8003424a33c8b84030224746495b9736c664347aafb1f81416`; +- execution runner SHA-256: + `728937c53c26ade2703014bd44bd440e82bd59e6dfebac65977a2a765dfd209f`; and +- frozen skill SHA-256: + `0aec0785111a9e4b67d2323684d6372ccaba03b84ddbecd850d4b658df1cdee2`. ## Audit Trail -Evaluator-only case identities, gold, authorizations, provider transcripts, -workspaces, and per-run grading remain under the ignored private experiment -tree. They are intentionally not published with the protocol. The aggregate -values above were transcribed from the completed private scorecard whose -SHA-256 is: +Evaluator-only identities, gold, authorizations, transcripts, workspaces, +blinded dossiers, and per-run grading remain under the ignored private +experiment tree. They are intentionally not published with the protocol. + +The completed private scorecard has SHA-256: ```text -7741a127949b0bf8881b8827e1ec46ee12082a40c84f7bc8ba6b25ba409269b7 +789f8cae51ea57f541a938770e5cc12c5099690c02c736260b2300d0062a8f1a ``` -The private narrative report used for the transcription has SHA-256: +The private semantic report has SHA-256: ```text -cfbf53af662407bbe8df74970cbaf107573d74d3d3ce8881976f3103ca2de78d +33b769d1d093609786dc1ab5a12d062c18d86eed462a2035e750721ecba1ca02 ``` These hashes bind the public aggregate report to the preserved evaluator diff --git a/docs/book-replay-conformance.md b/docs/book-replay-conformance.md index db55b92..6627737 100644 --- a/docs/book-replay-conformance.md +++ b/docs/book-replay-conformance.md @@ -77,6 +77,10 @@ alias for `tracebook.conformance`. ## Run It +Book replay is unreleased and is not present in the published 0.6.0 wheel. +Install this source checkout (`python -m pip install -e .`) or its built wheel, +and record the exact source revision with any evidence. + Copy the bundled trace: ```bash diff --git a/docs/commands.md b/docs/commands.md index 86a11ea..fd20c4f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -133,6 +133,9 @@ fixed semantic surface. Prepare and verify a release-grade two-run evidence pair: +These two evidence commands are unreleased. Use a pinned source build, not the +published 0.6.0 wheel, and record its revision and wheel hash. + ```bash tracebook-conformance evidence-init /path/to/candidate \ --workspace .tracebook/release-evidence \ @@ -254,7 +257,10 @@ Options: ## Dashboard CLI +Install the optional dashboard dependencies before starting the interactive UI: + ```bash +python -m pip install "tracebook-sim[dashboard]==0.6.0" tracebook-dashboard --port 8050 --demo-simulation --demo-throughput 200 --seed 1337 ``` diff --git a/docs/conformance.md b/docs/conformance.md index 6e42336..bfae977 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -137,6 +137,11 @@ silently. ## Captured Two-Run Evidence +This workflow is unreleased. Use a pinned source checkout or a wheel built from +that checkout, recording its Git revision and SHA-256; the published 0.6.0 wheel +does not contain these commands. The source version string alone cannot +distinguish this build from the released package. + Use the evidence workflow when a qualification result will support a release, benchmark, or external claim. It pins the candidate before adapter readiness and requires the same canonical qualification from two prepared roots. diff --git a/docs/independent-onboarding.md b/docs/independent-onboarding.md new file mode 100644 index 0000000..b662c5f --- /dev/null +++ b/docs/independent-onboarding.md @@ -0,0 +1,145 @@ +# Independent Onboarding: Captured Go Evidence + +## Status And Scope + +This packet is prepared for one independent maintainer attempt against +`intrepidkarthi/orderbook` v0.26.0, source commit +`51d480cdb68b9989febb0b075d291cf891f425b3`. A Tracebook-owned rehearsal is not +independent onboarding, adoption, or an upstream decision. + +The request is deliberately narrow: can the maintainer reproduce and understand +one verified FIFO-limit evidence pair, and would they retain the result or its +CI check? The native unfillable-FOK rejection is a documented contract +difference, not a bug accusation or a request to change native matching. + +The captured evidence workflow and this adapter are unreleased. The published +0.6.0 wheel does not contain the new evidence commands. Use a wheel built from +the exact Tracebook checkout below, not `pip install tracebook-conformance==0.6.0`. +The built wheel still carries version 0.6.0: record its SHA-256 and source commit +so it cannot be confused with that release. Do not upload it to PyPI. + +## Reproduce From A Reviewed Checkout + +Prerequisites: Git, Go >=1.23.5, Python >=3.10, and a POSIX shell. Start at the +root of a clean, reviewed Tracebook checkout containing this packet. Review the +adapter before running it; candidate execution is not sandboxed. These commands +download only public source and Go modules, and never use experiment material. + +Create and inspect the conformance wheel following the repository's build and +privacy gates. Then set `WHEEL` to that artifact's absolute path. Run the +following with Bash; all candidate, cache, build, and qualification outputs stay +under a fresh directory, which is preserved on failure. + +```bash +set -euo pipefail +TRACEBOOK_ROOT=$PWD +TRACEBOOK_REV=$(git rev-parse HEAD) +test -z "$(git status --porcelain --untracked-files=no)" +WHEEL="$TRACEBOOK_ROOT/dist/conformance/tracebook_conformance-0.6.0-py3-none-any.whl" +test -f "$WHEEL" +shasum -a 256 "$WHEEL" + +ATTEMPT_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/tracebook-onboarding.XXXXXX") +export ATTEMPT_ROOT +python3 -m venv "$ATTEMPT_ROOT/tool" +PYTHON="$ATTEMPT_ROOT/tool/bin/python" +"$PYTHON" -m pip install "$WHEEL" +"$PYTHON" -m tracebook.conformance.cli --help + +CANDIDATE_REV=51d480cdb68b9989febb0b075d291cf891f425b3 +git clone --depth 1 --branch v0.26.0 \ + https://github.com/intrepidkarthi/orderbook.git "$ATTEMPT_ROOT/source" +test "$(git -C "$ATTEMPT_ROOT/source" rev-parse HEAD)" = "$CANDIDATE_REV" + +WORKSPACE="$ATTEMPT_ROOT/evidence" +"$PYTHON" -m tracebook.conformance.cli evidence-init "$ATTEMPT_ROOT/source" \ + --workspace "$WORKSPACE" \ + --candidate-name 'intrepidkarthi/orderbook CLOB' \ + --candidate-revision "$CANDIDATE_REV" +SNAPSHOT=$("$PYTHON" -c \ + 'import json,sys; print(json.load(open(sys.argv[1]))["candidate"]["snapshot_id"])' \ + "$WORKSPACE/evidence-plan.json") + +for RUN_NUMBER in 1 2; do + RUN_ROOT="$WORKSPACE/runs/run-$RUN_NUMBER" + git -C "$TRACEBOOK_ROOT" archive "$TRACEBOOK_REV" integrations/intrepid_orderbook \ + | tar -x --strip-components=2 -C "$RUN_ROOT/adapter" + ( + cd "$RUN_ROOT/adapter" + export GOWORK=off GOCACHE="$RUN_ROOT/cache/build" + export GOMODCACHE="$RUN_ROOT/cache/modules" GOPATH="$RUN_ROOT/cache/gopath" + go mod edit "-replace=github.com/intrepidkarthi/orderbook=$RUN_ROOT/candidate" + go mod download + go build -mod=readonly -trimpath \ + -ldflags "-X main.engineRevision=$CANDIDATE_REV -X main.engineSnapshot=$SNAPSHOT" \ + -o "$RUN_ROOT/build/adapter" . + go version -m "$RUN_ROOT/build/adapter" + ) + "$PYTHON" -m tracebook.conformance.cli qualify \ + --profile fifo-limit-v1 --suite-version v2 \ + --seed 42 --traces 25 --events-per-trace 200 --max-minimize-runs 100 \ + --timeout 10 \ + --candidate-name 'intrepidkarthi/orderbook CLOB' \ + --candidate-revision "$CANDIDATE_REV" --candidate-snapshot "$SNAPSHOT" \ + --output-dir "$RUN_ROOT/qualification" \ + --candidate "$RUN_ROOT/build/adapter" +done + +"$PYTHON" -m tracebook.conformance.cli evidence-verify "$WORKSPACE/evidence-plan.json" +``` + +Expected result: each run passes 3/3 fixed cases, 25/25 generated traces, 5,000 +generated events, and 10/10 capabilities. Verification emits +`evidence-manifest.json` only when both candidate trees remain unchanged and +their canonical bundles are byte-identical. Identity-bound qualification IDs +differ from the integration README's unbound runs; do not substitute one for +the other. + +The source replacement above is essential: embedding a snapshot ID in an +otherwise ordinary module-cache build does not prove that captured source was +used. Each run gets its own module and compiler caches. This remains an +auditable process, not a hermetic build or proof against a dishonest adapter; +see the [evidence trust boundary](conformance.md#captured-two-run-evidence). + +## Measurement Record + +Before the independent attempt, record its operator, relationship to the engine, +start time, Tracebook commit and wheel hash, candidate commit and snapshot, +operating system, Python and Go versions. Start the clock before environment +setup; do not exclude dependency downloads or time spent asking questions. + +Record each failed command and each human correction, including who suggested +it, what changed, and minutes spent. Preserve failed outputs in separate roots; +do not edit a failed bundle into a passing one. Record elapsed time to the first +verified pair or the first blocker, even if no pair succeeds. A provided adapter +measures time to evidence, not time to author an adapter. + +The result record must contain: + +- operator and relationship: **not yet observed**; +- first verified pair or blocker time: **not yet observed**; +- failed attempts and human corrections: **not yet observed**; +- plan, manifest, qualification and campaign IDs: **not yet observed**; +- semantic questions or disagreements: **not yet observed**; and +- retention decision (retained, declined, undecided) plus a public link if + retained: **not yet observed**. + +An external maintainer running the packet establishes independent use, not +retention. A promise to add CI is not retained CI. A Tracebook-owned run or job +is neither. Report the attempt through the existing Engine qualification issue +form, adding the wheel hash, evidence manifest and human-correction log. Remove +credentials, proprietary traces and machine-specific paths before publication. + +## Proposed Maintainer Invitation (Not Sent) + +> We have a pinned, out-of-tree Tracebook adapter for your v0.26.0 CLOB. FIFO +> limit qualification passes; the broader FOK profile stops on your native +> unfillable-FOK rejection, which we preserve as a contract difference, not a +> defect. Would you be willing to try one captured two-run FIFO evidence packet +> and tell us whether its result is useful enough to retain? No matching-engine +> change or in-tree adapter commitment is requested. We would record setup +> time, corrections, and your retention decision, including a failed attempt. + +Attach the reviewed Tracebook source revision and artifact hashes before +sending. A maintainer response and any independent result must be recorded as +new evidence; this prepared invitation establishes neither. diff --git a/integrations/intrepid_orderbook/README.md b/integrations/intrepid_orderbook/README.md index 70d1895..03f28ea 100644 --- a/integrations/intrepid_orderbook/README.md +++ b/integrations/intrepid_orderbook/README.md @@ -26,16 +26,58 @@ engine's matching decisions. The protocol configuration maps FIFO and pro-rata allocation plus `NONE`, `CANCEL_RESTING`, and `CANCEL_INCOMING` self-trade policies to their native -settings. Prices are converted exactly to integer ticks. Quantities are first -normalized at the protocol precision and then represented as integer native -lots. The adapter accepts at most six non-zero quantity decimal places because -the upstream pro-rata implementation multiplies two `int64` lot values; values -that cannot be represented exactly at that boundary are explicitly rejected. +settings. The anonymous owner sentinel (`-1`, also used when owner is omitted) +maps to a distinct native account for each active source order, so anonymous +orders remain free to trade under either STP policy. Snapshots retain the +original owner ID, including across replace and reduce operations. + +Prices use binary64 division and ties-to-even tick snapping, matching the +reference's input boundary, and are formatted using the exact decimal tick +size. Quantities are represented exactly in native integer lots at six decimal +places. `quantity_decimal_places` rounds output quantities only; it never +rounds the quantity submitted to the engine. Inputs requiring more than six +non-zero decimal places are explicitly rejected even if the configured output +precision is lower. + +The native arithmetic has a bounded domain: + +- price ticks and scaled quantities must be positive and fit signed `int64`; +- before admitting a limit order, the adapter requires its full quantity plus + all resting quantity on its side to fit signed `int64` (replacement excludes + the old order); +- under pro-rata, incoming quantity times each potentially crossed maker's + remaining quantity must fit signed `int64`; and +- the native engine's own price-times-quantity notional limit remains in force + for limit, IOC, and FOK instructions; the adapter checks it before native + submission or replacement so an invalid replacement cannot cancel its old + order. Market instructions retain the native exemption. + +The aggregate and pro-rata checks are conservative: they reserve the full +incoming quantity without predicting fills or STP removals. An input outside +this envelope returns `INVALID_ORDER` (or `INVALID_REPLACEMENT`) before native +mutation; no claim of conformance is made beyond that envelope. At the default +scale, for example, a 4,000-unit order crossing another 4,000-unit order is +rejected under pro-rata because their native lot product exceeds `int64`. +The same FIFO crossing is supported. + +Numeric configurations outside the recorded profiles remain unqualified even +when their values fit that arithmetic envelope. The adapter preserves native +integer/decimal results rather than recreating the reference's floating-point +arithmetic. At `quantity_decimal_places=18`, consuming `0.1` from `0.3` leaves +native quantity `0.2`, while the binary64 reference reports +`0.19999999999999998`. With tick size `1e-18`, input price `1` snaps to native +integer tick `999999999999999872`, formatted as `0.999999999999999872`; the +reference's floating-point price formatting produces `0.9999999999999999`. +These configurations are accepted for investigation, but can diverge and have +not passed qualification. Tests retain those native outputs explicitly. The local Go tests include a submitted-order crossing test: two same-price bids are reduced and then consumed by an incoming sell in native FIFO order. That guards the central claim that this is a matching-engine integration, not an order-book mirror or a reimplemented reference engine. +Additional tests cover anonymous STP, replacement owner retention, fractional +input quantities with zero-place output, binary64 tick boundaries, and native +arithmetic limits including rejection without replacing an existing order. ## Qualification Results @@ -92,6 +134,20 @@ tracebook-conformance qualify \ --output-dir /tmp/intrepid-fifo-limit ``` +For the identity-bound evidence workflow, capture the candidate source snapshot +first and inject its recorded revision and snapshot ID when building: + +```bash +go build -trimpath \ + -ldflags "-X main.engineRevision=$CANDIDATE_REVISION -X main.engineSnapshot=$CANDIDATE_SNAPSHOT_ID" \ + -o /tmp/tracebook-intrepid-orderbook . +``` + +These optional build variables appear as `engine.revision` and +`engine.snapshot_id` in the ready handshake. Both are omitted from ordinary +builds. They must describe the captured source used for that build; the adapter +does not discover or verify the source snapshot itself. + The tag is reproducible evidence, not a production-readiness certification. This integration makes no claim about upstream adoption, operational maturity, latency, durability, or suitability for live trading. diff --git a/integrations/intrepid_orderbook/adapter.go b/integrations/intrepid_orderbook/adapter.go index b3cf34d..51a642a 100644 --- a/integrations/intrepid_orderbook/adapter.go +++ b/integrations/intrepid_orderbook/adapter.go @@ -17,6 +17,7 @@ import ( type adapterConfig struct { matchingAlgorithm string tickSize decimal.Decimal + tickSizeFloat float64 quantityPlaces uint32 nativeQtyPlaces uint32 quantityScale int64 @@ -39,6 +40,10 @@ func parseConfig(wire configWire) (adapterConfig, error) { if err != nil || !tick.IsPositive() { return adapterConfig{}, errors.New("tick_size must be a positive finite decimal") } + tickFloat, err := strconv.ParseFloat(tickText, 64) + if err != nil || math.IsInf(tickFloat, 0) || math.IsNaN(tickFloat) || tickFloat <= 0 { + return adapterConfig{}, errors.New("tick_size must be a positive finite binary64 value") + } places := uint32(12) if wire.QuantityDecimalPlaces != nil { places = *wire.QuantityDecimalPlaces @@ -46,18 +51,11 @@ func parseConfig(wire configWire) (adapterConfig, error) { if places > 18 { return adapterConfig{}, errors.New("quantity_decimal_places must be between 0 and 18") } - // The upstream pro-rata path multiplies two int64 lot counts. Scaling all - // protocol quantities by 10^12 would overflow that native multiplication on - // ordinary five-lot fixtures. Six decimal places preserve the campaign's - // exact values while keeping its bounded native arithmetic safely in range. - nativePlaces := min(places, uint32(6)) - scale := int64(1) - for range nativePlaces { - if scale > math.MaxInt64/10 { - return adapterConfig{}, errors.New("quantity scale exceeds int64") - } - scale *= 10 - } + // Protocol precision controls observations, not submitted quantities. + // Keep six native decimal places regardless of that output setting, and + // reject inputs that would require rounding before the engine sees them. + const nativePlaces = uint32(6) + const scale = int64(1_000_000) policy := wire.SelfTradePolicy if policy == "" { policy = "NONE" @@ -75,26 +73,30 @@ func parseConfig(wire configWire) (adapterConfig, error) { "self_trade_policy must be NONE, CANCEL_RESTING, or CANCEL_INCOMING", ) } - return adapterConfig{algorithm, tick, places, nativePlaces, scale, stp}, nil + return adapterConfig{ + matchingAlgorithm: algorithm, tickSize: tick, tickSizeFloat: tickFloat, + quantityPlaces: places, nativeQtyPlaces: nativePlaces, quantityScale: scale, stp: stp, + }, nil } func (config adapterConfig) priceTicks(value *jsonNumber) (int64, error) { if value == nil { return 0, errors.New("price is required") } - price, err := decimal.NewFromString(value.String()) - if err != nil { + price, err := strconv.ParseFloat(value.String(), 64) + if err != nil || math.IsInf(price, 0) || math.IsNaN(price) { return 0, errors.New("price must be finite") } - ticks := price.Div(config.tickSize).RoundBank(0).BigInt() - if !ticks.IsInt64() || ticks.Sign() <= 0 { + // The reference snaps the normalized binary64 input with ties to even. + // Decimal division differs at boundaries such as 1.015 / 0.01. + ticks := math.RoundToEven(price / config.tickSizeFloat) + if math.IsInf(ticks, 0) || math.IsNaN(ticks) || ticks <= 0 || ticks >= math.Exp2(63) { return 0, errors.New("price snaps to a non-positive or unsupported tick") } - return ticks.Int64(), nil + return int64(ticks), nil } -// jsonNumber is an alias so conversion helpers read cleanly without allowing -// floating-point parsing into this integer-tick boundary. +// jsonNumber preserves the wire text for each explicit numeric conversion. type jsonNumber = json.Number func (config adapterConfig) quantityUnits(value *jsonNumber) (int64, error) { @@ -105,8 +107,7 @@ func (config adapterConfig) quantityUnits(value *jsonNumber) (int64, error) { if err != nil || !quantity.IsPositive() { return 0, errors.New("quantity must be positive") } - normalized := quantity.RoundBank(int32(config.quantityPlaces)) - scaled := normalized.Mul( + scaled := quantity.Mul( decimal.NewFromInt(config.quantityScale), ) if !scaled.Equal(scaled.Truncate(0)) { @@ -127,13 +128,15 @@ func (config adapterConfig) formatPrice(ticks int64) string { } func (config adapterConfig) formatQuantity(units int64) string { - return decimal.NewFromInt(units).Div(decimal.NewFromInt(config.quantityScale)).String() + return decimal.NewFromInt(units).Shift(-int32(config.nativeQtyPlaces)). + RoundBank(int32(config.quantityPlaces)).String() } type bookHarness struct { engine *matching.Engine sourceToNative map[int64]int64 nativeToSource map[int64]int64 + nativeToOwner map[int64]int64 } func newBook(symbol string, config adapterConfig) *bookHarness { @@ -145,6 +148,7 @@ func newBook(symbol string, config adapterConfig) *bookHarness { engine: matching.NewEngine(nativeConfig), sourceToNative: make(map[int64]int64), nativeToSource: make(map[int64]int64), + nativeToOwner: make(map[int64]int64), } } @@ -156,6 +160,7 @@ func (book *bookHarness) reconcile() { for nativeID, sourceID := range book.nativeToSource { if !active[nativeID] { delete(book.nativeToSource, nativeID) + delete(book.nativeToOwner, nativeID) if book.sourceToNative[sourceID] == nativeID { delete(book.sourceToNative, sourceID) } @@ -203,6 +208,56 @@ func eventOwner(event marketEvent) int64 { return *event.Owner } +func nativeOwner(sourceID, owner int64) string { + if owner == -1 { + // Anonymous source orders never self-match. Give each active anonymous + // order its own native account, disjoint from every numeric owner id. + return "anonymous:" + strconv.FormatInt(sourceID, 10) + } + return strconv.FormatInt(owner, 10) +} + +// checkQuantityCapacity rejects unsupported native arithmetic before mutation. +// It reserves the entire incoming limit quantity on its side, even if some +// would execute immediately, and checks each potentially crossed pro-rata +// maker conservatively using the entire incoming quantity. This is a numeric +// envelope, not a replacement for the engine's liquidity or matching decisions. +func (book *bookHarness) checkQuantityCapacity( + config adapterConfig, side types.Side, price, quantity int64, + orderType string, replacedID int64, +) error { + // Process checks this for native limit orders (including IOC/FOK). Repeat + // the same representability check before Replace can cancel its old order. + // Market orders have no submitted notional and retain the native exemption. + if orderType != "MARKET" && price > math.MaxInt64/quantity { + return types.ErrNotionalOverflow + } + var sameSideTotal int64 + for _, resting := range book.engine.RestingOrders() { + if resting.ID == replacedID { + continue + } + if resting.Side == side { + if resting.RemainingQty > math.MaxInt64-sameSideTotal { + return errors.New("resting quantity exceeds the native int64 aggregate range") + } + sameSideTotal += resting.RemainingQty + continue + } + crosses := orderType == "MARKET" || + (side == types.SideBuy && price >= resting.Price) || + (side == types.SideSell && price <= resting.Price) + if config.matchingAlgorithm == "pro_rata" && crosses && + quantity > math.MaxInt64/resting.RemainingQty { + return errors.New("quantity exceeds the native int64 pro-rata product range") + } + } + if orderType == "LIMIT" && quantity > math.MaxInt64-sameSideTotal { + return errors.New("quantity exceeds the native int64 side aggregate range") + } + return nil +} + func nativeSide(value *string) (types.Side, error) { if value == nil { return "", errors.New("side is required") @@ -337,7 +392,11 @@ func (adapter *adapter) applyNew( default: return rejected("INVALID_ORDER", "unsupported order_type"), nil, nil } - owner := strconv.FormatInt(eventOwner(event), 10) + if err := book.checkQuantityCapacity(adapter.config, side, price, quantity, orderType, 0); err != nil { + return rejected("INVALID_ORDER", err.Error()), nil, nil + } + sourceOwner := eventOwner(event) + owner := nativeOwner(source, sourceOwner) order, err := types.NewOrder(owner, event.Symbol, side, nativeType, price, quantity, tif) if err != nil { return rejected("INVALID_ORDER", err.Error()), nil, nil @@ -345,6 +404,7 @@ func (adapter *adapter) applyNew( order.ClientOrderID = strconv.FormatInt(source, 10) match := book.engine.Process(order) book.nativeToSource[order.ID] = source + book.nativeToOwner[order.ID] = sourceOwner converted, err := adapter.convertTrades(book, event.Symbol, match.Trades) if err != nil { return outcome{}, nil, err @@ -388,6 +448,10 @@ func (adapter *adapter) applyLifecycle( return rejected("ORDER_NOT_ACTIVE", "order is not active"), nil, nil } owner := current.UserID + sourceOwner, ok := book.nativeToOwner[nativeID] + if !ok { + return outcome{}, nil, fmt.Errorf("adapter has no source owner for native order %d", nativeID) + } switch event.Op { case "cancel": if _, err := book.engine.Cancel(nativeID, owner); err != nil { @@ -431,6 +495,11 @@ func (adapter *adapter) applyLifecycle( return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil } } + if err := book.checkQuantityCapacity( + adapter.config, current.Side, price, quantity, "LIMIT", nativeID, + ); err != nil { + return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil + } replacement, err := types.NewOrder( owner, event.Symbol, current.Side, types.OrderTypeLimit, price, quantity, types.TIFGoodTillCancel, @@ -444,6 +513,7 @@ func (adapter *adapter) applyLifecycle( return rejected("INVALID_REPLACEMENT", err.Error()), nil, nil } book.nativeToSource[replacement.ID] = source + book.nativeToOwner[replacement.ID] = sourceOwner converted, err := adapter.convertTrades(book, event.Symbol, match.Trades) if err != nil { return outcome{}, nil, err @@ -484,12 +554,11 @@ func (adapter *adapter) snapshot() (bookState, error) { native.ID, ) } - owner, err := strconv.ParseInt(native.UserID, 10, 64) - if err != nil { + owner, ok := book.nativeToOwner[native.ID] + if !ok { return bookState{}, fmt.Errorf( - "adapter has invalid source owner for native order %d: %w", + "adapter has no source owner for native order %d", native.ID, - err, ) } resting := restingOrder{ diff --git a/integrations/intrepid_orderbook/adapter_test.go b/integrations/intrepid_orderbook/adapter_test.go index 2686ba1..b59400e 100644 --- a/integrations/intrepid_orderbook/adapter_test.go +++ b/integrations/intrepid_orderbook/adapter_test.go @@ -3,6 +3,8 @@ package main import ( "bytes" "encoding/json" + "math" + "reflect" "strings" "testing" ) @@ -16,6 +18,29 @@ func textPointer(value string) *string { return &value } func int64Pointer(value int64) *int64 { return &value } +func testAdapter(t *testing.T, config configWire) *adapter { + t.Helper() + harness, err := newAdapter(config) + if err != nil { + t.Fatal(err) + } + return harness +} + +func applyForTest(t *testing.T, harness *adapter, event marketEvent) observationFrame { + t.Helper() + observation, err := harness.apply(event, 1) + if err != nil { + t.Fatal(err) + } + return observation +} + +func limitEvent(id, side, price, quantity string) marketEvent { + return marketEvent{Op: "new", Symbol: "TEST", OrderID: number(id), + Side: textPointer(side), OrderType: "LIMIT", Price: number(price), Quantity: number(quantity)} +} + func TestEmptyStateDigestMatchesProtocolCanonicalJSON(t *testing.T) { digest, err := (bookState{Books: []bookSnapshot{}}).digest() if err != nil { @@ -137,7 +162,324 @@ func TestServerHandshakeAndEmptyCompletion(t *testing.T) { if ready["type"] != "ready" || engine["name"] != engineName || engine["version"] != engineVersion { t.Fatalf("ready = %#v", ready) } + if _, ok := engine["revision"]; ok { + t.Fatal("default handshake unexpectedly binds a revision") + } + if _, ok := engine["snapshot_id"]; ok { + t.Fatal("default handshake unexpectedly binds a snapshot") + } if complete["type"] != "complete" || complete["event_count"] != float64(0) { t.Fatalf("complete = %#v", complete) } } + +func TestServerHandshakeReportsInjectedEvidenceIdentity(t *testing.T) { + previousRevision, previousSnapshot := engineRevision, engineSnapshot + t.Cleanup(func() { + engineRevision, engineSnapshot = previousRevision, previousSnapshot + }) + engineRevision = "reviewed-source-revision" + engineSnapshot = "sha256:captured-source-snapshot" + input := `{"type":"hello","protocol":"tracebook.conformance","protocol_version":1,"config":{}} +{"type":"finish","event_count":0}` + var output bytes.Buffer + if status := serve(strings.NewReader(input), &output); status != 0 { + t.Fatalf("status = %d, output = %s", status, output.String()) + } + var ready struct { + Engine map[string]string `json:"engine"` + } + if err := json.NewDecoder(&output).Decode(&ready); err != nil { + t.Fatal(err) + } + if ready.Engine["revision"] != engineRevision || ready.Engine["snapshot_id"] != engineSnapshot || + ready.Engine["name"] != engineName || ready.Engine["version"] != engineVersion { + t.Fatalf("bound identity = %#v", ready.Engine) + } +} + +func TestAnonymousOwnersRemainExemptFromNativeSTPAfterReplace(t *testing.T) { + for _, algorithm := range []string{"fifo", "pro_rata"} { + for _, policy := range []string{"CANCEL_RESTING", "CANCEL_INCOMING"} { + for _, test := range []struct { + name string + makerOwner *int64 + takerOwner *int64 + selfMatch bool + }{ + {"omitted", nil, nil, false}, + {"explicit-anonymous", int64Pointer(-1), int64Pointer(-1), false}, + {"mixed-anonymous", nil, int64Pointer(-1), false}, + {"anonymous-vs-numeric", int64Pointer(-1), int64Pointer(1), false}, + {"same-numeric", int64Pointer(1), int64Pointer(1), true}, + {"same-negative-numeric", int64Pointer(-2), int64Pointer(-2), true}, + } { + t.Run(algorithm+"/"+policy+"/"+test.name, func(t *testing.T) { + harness := testAdapter(t, configWire{MatchingAlgorithm: algorithm, SelfTradePolicy: policy}) + maker := limitEvent("1", "BUY", "100", "2") + maker.Owner = test.makerOwner + for _, event := range []marketEvent{ + maker, + {Op: "replace", Symbol: "TEST", OrderID: number("1"), Quantity: number("3")}, + {Op: "reduce", Symbol: "TEST", OrderID: number("1"), Quantity: number("1")}, + } { + if got := applyForTest(t, harness, event); got.Outcome.Status != "applied" { + t.Fatalf("%s outcome = %#v", event.Op, got.Outcome) + } + } + state, err := harness.snapshot() + if err != nil { + t.Fatal(err) + } + if got := state.Books[0].Bids[0].Owner; got != eventOwner(maker) { + t.Fatalf("owner after replace/reduce = %d, want %d", got, eventOwner(maker)) + } + taker := limitEvent("2", "SELL", "100", "1") + taker.Owner = test.takerOwner + got := applyForTest(t, harness, taker) + if got.Outcome.Status != "applied" { + t.Fatalf("taker outcome = %#v", got.Outcome) + } + if !test.selfMatch { + if len(got.Trades) != 1 || got.Trades[0].BuyOrderID != 1 || got.Trades[0].Quantity != "1" { + t.Fatalf("anonymous crossing = %#v, want a native fill", got) + } + } else if len(got.Trades) != 0 { + t.Fatalf("same-owner order traded under %s: %#v", policy, got.Trades) + } + activeID := "1" + if test.selfMatch && policy == "CANCEL_RESTING" { + activeID = "2" + } + cancel := applyForTest(t, harness, marketEvent{Op: "cancel", Symbol: "TEST", OrderID: number(activeID)}) + if cancel.Outcome.Status != "applied" || cancel.RestingOrderCount != 0 { + t.Fatalf("cancel after STP = %#v", cancel) + } + book := harness.books["TEST"] + if len(book.nativeToOwner) != 0 || len(book.nativeToSource) != 0 || len(book.sourceToNative) != 0 { + t.Fatal("inactive source and owner mappings were retained") + } + }) + } + } + } +} + +func TestQuantityPrecisionOnlyRoundsObservations(t *testing.T) { + places := uint32(0) + harness := testAdapter(t, configWire{QuantityDecimalPlaces: &places}) + first := applyForTest(t, harness, limitEvent("1", "BUY", "100", "2.4")) + if first.Outcome.Status != "applied" { + t.Fatalf("first outcome = %#v", first.Outcome) + } + if got := harness.books["TEST"].engine.RestingOrders()[0].RemainingQty; got != 2_400_000 { + t.Fatalf("native quantity = %d, want the unrounded input 2400000", got) + } + second := applyForTest(t, harness, limitEvent("2", "SELL", "100", "3.5")) + if second.Outcome.Status != "applied" || len(second.Trades) != 1 || second.Trades[0].Quantity != "2" { + t.Fatalf("crossing = %#v, want the native 2.4 fill reported at zero decimal places", second) + } + state, err := harness.snapshot() + if err != nil { + t.Fatal(err) + } + if len(state.Books[0].Asks) != 1 || state.Books[0].Asks[0].RemainingQuantity != "1" { + t.Fatalf("state = %#v, want 1.1 native quantity reported as 1", state) + } + if got := harness.books["TEST"].engine.RestingOrders()[0].RemainingQty; got != 1_100_000 { + t.Fatalf("native remainder = %d, want 1100000", got) + } + if _, err := harness.config.quantityUnits(number("1.0000001")); err == nil { + t.Fatal("input beyond six native places must not be rounded into range") + } +} + +func TestPriceTicksMatchReferenceBinary64Snapping(t *testing.T) { + config := testAdapter(t, configWire{}).config + for _, test := range []struct { + price string + ticks int64 + }{ + {"1.015", 101}, // Binary64 division is just below 101.5. + {"0.105", 10}, + {"0.115", 12}, + {"100.004", 10000}, + {"100.006", 10001}, + } { + t.Run(test.price, func(t *testing.T) { + got, err := config.priceTicks(number(test.price)) + if err != nil || got != test.ticks { + t.Fatalf("ticks = %d, %v; want %d", got, err, test.ticks) + } + }) + } + for _, price := range []string{"0.005", "0", "-1", "NaN", "Inf", "1e309", "92233720368547758.08"} { + if _, err := config.priceTicks(number(price)); err == nil { + t.Errorf("unsupported price %s was accepted", price) + } + } + for _, tick := range []string{"NaN", "Inf", "1e309", "1e-330"} { + if _, err := parseConfig(configWire{TickSize: tick}); err == nil { + t.Errorf("unsupported tick %s was accepted", tick) + } + } +} + +func TestNativeQuantityIntegerBoundary(t *testing.T) { + config := testAdapter(t, configWire{}).config + got, err := config.quantityUnits(number("9223372036854.775807")) + if err != nil || got != math.MaxInt64 { + t.Fatalf("maximum native quantity = %d, %v", got, err) + } + for _, quantity := range []string{"9223372036854.775808", "0", "-1", "NaN"} { + if _, err := config.quantityUnits(number(quantity)); err == nil { + t.Errorf("unsupported quantity %s was accepted", quantity) + } + } +} + +func TestUnsafeProRataProductIsRejectedBeforeMutation(t *testing.T) { + for _, orderType := range []string{"LIMIT", "MARKET", "IOC", "FOK"} { + t.Run(orderType, func(t *testing.T) { + harness := testAdapter(t, configWire{MatchingAlgorithm: "pro_rata"}) + maker := applyForTest(t, harness, limitEvent("1", "SELL", "100", "4000")) + taker := limitEvent("2", "BUY", "100", "4000") + taker.OrderType = orderType + got := applyForTest(t, harness, taker) + if got.Outcome.Status != "rejected" || got.Outcome.Message == nil || + !strings.Contains(*got.Outcome.Message, "pro-rata product range") { + t.Fatalf("unsafe product outcome = %#v", got.Outcome) + } + if got.StateHash != maker.StateHash || len(got.Trades) != 0 { + t.Fatalf("unsafe product mutated state: %#v", got) + } + }) + } + for _, test := range []struct{ algorithm, quantity string }{ + {"fifo", "4000"}, {"pro_rata", "3000"}, + } { + t.Run(test.algorithm+"/safe-product", func(t *testing.T) { + harness := testAdapter(t, configWire{MatchingAlgorithm: test.algorithm}) + applyForTest(t, harness, limitEvent("1", "SELL", "100", test.quantity)) + got := applyForTest(t, harness, limitEvent("2", "BUY", "100", test.quantity)) + if got.Outcome.Status != "applied" || len(got.Trades) != 1 || got.Trades[0].Quantity != test.quantity { + t.Fatalf("supported large crossing = %#v", got) + } + }) + } +} + +func TestSideAggregateAndReplacementCapacityPreserveExistingOrders(t *testing.T) { + harness := testAdapter(t, configWire{}) + first := applyForTest(t, harness, limitEvent("1", "BUY", "0.01", "4700000000000")) + if first.Outcome.Status != "applied" { + t.Fatalf("large first order = %#v", first) + } + tooLarge := applyForTest(t, harness, limitEvent("2", "BUY", "0.01", "4700000000000")) + if tooLarge.Outcome.Status != "rejected" || tooLarge.StateHash != first.StateHash || + tooLarge.Outcome.Message == nil || !strings.Contains(*tooLarge.Outcome.Message, "side aggregate range") { + t.Fatalf("aggregate overflow = %#v", tooLarge) + } + second := applyForTest(t, harness, limitEvent("2", "BUY", "0.01", "4000000000000")) + if second.Outcome.Status != "applied" { + t.Fatalf("supported aggregate = %#v", second) + } + replace := applyForTest(t, harness, marketEvent{Op: "replace", Symbol: "TEST", + OrderID: number("2"), Quantity: number("4700000000000")}) + if replace.Outcome.Reason == nil || *replace.Outcome.Reason != "INVALID_REPLACEMENT" || + replace.StateHash != second.StateHash { + t.Fatalf("unsafe replacement = %#v", replace) + } + noChange := applyForTest(t, harness, marketEvent{Op: "replace", Symbol: "TEST", + OrderID: number("2"), Quantity: number("4000000000000")}) + if noChange.Outcome.Status != "applied" { + t.Fatalf("replacement must exclude the old quantity from its capacity check: %#v", noChange) + } +} + +func TestNativeNotionalLimitAppliesOnlyToPriceBearingInstructions(t *testing.T) { + for _, orderType := range []string{"LIMIT", "IOC", "FOK", "MARKET"} { + t.Run(orderType, func(t *testing.T) { + harness := testAdapter(t, configWire{}) + event := limitEvent("1", "BUY", "100000", "1000000") + event.OrderType = orderType + got := applyForTest(t, harness, event) + if orderType == "MARKET" { + if got.Outcome.Status != "applied" || got.RestingOrderCount != 0 { + t.Fatalf("market order should retain the native notional exemption: %#v", got) + } + } else if got.Outcome.Status != "rejected" || got.Outcome.Message == nil || + !strings.Contains(*got.Outcome.Message, "order notional") || got.RestingOrderCount != 0 { + t.Fatalf("native notional overflow = %#v", got) + } + }) + } +} + +func TestNotionalOverflowReplacementPreservesNativeOrderAndSourceMetadata(t *testing.T) { + for _, test := range []struct { + name string + owner *int64 + }{{"anonymous", nil}, {"identified", int64Pointer(8)}} { + t.Run(test.name, func(t *testing.T) { + harness := testAdapter(t, configWire{}) + event := limitEvent("1", "BUY", "100", "1") + event.Owner = test.owner + first := applyForTest(t, harness, event) + book := harness.books["TEST"] + original := *book.engine.RestingOrders()[0] + got := applyForTest(t, harness, marketEvent{ + Op: "replace", Symbol: "TEST", OrderID: number("1"), + Price: number("100000"), Quantity: number("1000000"), + }) + if got.Outcome.Reason == nil || *got.Outcome.Reason != "INVALID_REPLACEMENT" || + got.Outcome.Message == nil || !strings.Contains(*got.Outcome.Message, "order notional") { + t.Fatalf("overflow replacement outcome = %#v", got.Outcome) + } + if got.StateHash != first.StateHash || got.RestingOrderCount != 1 || len(got.Trades) != 0 { + t.Fatalf("overflow replacement changed the source state: %#v", got) + } + orders := book.engine.RestingOrders() + if len(orders) != 1 || !reflect.DeepEqual(original, *orders[0]) { + t.Fatalf("overflow replacement changed the native order: %#v", orders) + } + if len(book.sourceToNative) != 1 || book.sourceToNative[1] != original.ID || + len(book.nativeToSource) != 1 || book.nativeToSource[original.ID] != 1 || + len(book.nativeToOwner) != 1 || book.nativeToOwner[original.ID] != eventOwner(event) { + t.Fatal("overflow replacement changed source identity metadata") + } + }) + } +} + +func TestHighOutputPrecisionPreservesNativeDecimalRemainder(t *testing.T) { + places := uint32(18) + harness := testAdapter(t, configWire{QuantityDecimalPlaces: &places}) + applyForTest(t, harness, limitEvent("1", "BUY", "100", "0.3")) + got := applyForTest(t, harness, limitEvent("2", "SELL", "100", "0.1")) + state, err := harness.snapshot() + if err != nil { + t.Fatal(err) + } + // The binary64 reference retains 0.19999999999999998 at 18 places. + // Preserve the native arithmetic here so this known difference is visible. + if got.Outcome.Status != "applied" || len(state.Books[0].Bids) != 1 || + state.Books[0].Bids[0].RemainingQuantity != "0.2" { + t.Fatalf("native decimal remainder was not preserved: %#v", state) + } +} + +func TestFineTickOutputPreservesNativeIntegerPrice(t *testing.T) { + harness := testAdapter(t, configWire{TickSize: "0.000000000000000001"}) + got := applyForTest(t, harness, limitEvent("1", "BUY", "1", "0.000001")) + state, err := harness.snapshot() + if err != nil { + t.Fatal(err) + } + // The reference multiplies and rounds in binary64, yielding + // 0.9999999999999999. Native tick output must not imitate that arithmetic. + if got.Outcome.Status != "applied" || len(state.Books[0].Bids) != 1 || + state.Books[0].Bids[0].Price != "0.999999999999999872" { + t.Fatalf("native fine-grid price was not preserved: %#v", state) + } +} diff --git a/integrations/intrepid_orderbook/protocol.go b/integrations/intrepid_orderbook/protocol.go index 5dba8b9..6c08c55 100644 --- a/integrations/intrepid_orderbook/protocol.go +++ b/integrations/intrepid_orderbook/protocol.go @@ -19,6 +19,23 @@ const ( engineVersion = "v0.26.0@51d480cdb68b" ) +// Optional build-time identity binds an evidence run to its captured source. +// Set with -ldflags '-X main.engineRevision=... -X main.engineSnapshot=...'. +// Empty defaults preserve the unbound qualification handshake. +var engineRevision string +var engineSnapshot string + +func engineMetadata() map[string]string { + metadata := map[string]string{"name": engineName, "version": engineVersion, "language": "Go"} + if engineRevision != "" { + metadata["revision"] = engineRevision + } + if engineSnapshot != "" { + metadata["snapshot_id"] = engineSnapshot + } + return metadata +} + type configWire struct { MatchingAlgorithm string `json:"matching_algorithm"` TickSize string `json:"tick_size"` @@ -267,9 +284,7 @@ func runServer(decoder *json.Decoder, writer *bufio.Writer) error { "type": "ready", "protocol": protocolName, "protocol_version": protocolVersion, - "engine": map[string]string{ - "name": engineName, "version": engineVersion, "language": "Go", - }, + "engine": engineMetadata(), }); err != nil { return adapterError(err) } diff --git a/integrations/nautilus_trader/README.md b/integrations/nautilus_trader/README.md index 6beba86..6db3b56 100644 --- a/integrations/nautilus_trader/README.md +++ b/integrations/nautilus_trader/README.md @@ -57,6 +57,7 @@ python3.14 -m venv /tmp/tracebook-nautilus /tmp/tracebook-nautilus/bin/python -m tracebook.book_replay run \ src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl \ --output /tmp/nautilus-book-replay.json \ + --timeout 10 \ --candidate /tmp/tracebook-nautilus/bin/python \ integrations/nautilus_trader/adapter.py ``` @@ -76,6 +77,12 @@ The command exits `0` and the pinned candidate produces: } ``` +The timeout also bounds the candidate's natural exit after its `complete` +frame. A nonzero exit or a forced shutdown is a protocol failure, even if all +event observations matched. The integration CI installs `pytest` explicitly +before running the book-replay and integration contract tests; the base package +does not include test dependencies. + ## Negative Control And Retained Evidence [`faulty_adapter.py`](faulty_adapter.py) subclasses the real pinned native diff --git a/src/tracebook/book_replay/external.py b/src/tracebook/book_replay/external.py index 567922d..14101ae 100644 --- a/src/tracebook/book_replay/external.py +++ b/src/tracebook/book_replay/external.py @@ -243,19 +243,28 @@ def close(self) -> None: ) finally: self._closed = True - self._shutdown() - if not was_broken and close_error is None and self._process.returncode != 0: - close_error = BookReplayProtocolError( - f"candidate exited with code {self._process.returncode} after complete" + shutdown_timed_out = self._shutdown( + timeout_seconds=self.timeout_seconds if not self._broken else 0.5 ) + if not was_broken and close_error is None: + if shutdown_timed_out: + close_error = BookReplayProtocolError( + f"candidate timed out exiting after complete ({self.timeout_seconds:g}s)" + ) + elif self._process.returncode != 0: + close_error = BookReplayProtocolError( + f"candidate exited with code {self._process.returncode} after complete" + ) if close_error is not None: raise close_error - def _shutdown(self) -> None: + def _shutdown(self, *, timeout_seconds: float = 0.5) -> bool: + timed_out = False if self._process.poll() is None: try: - self._process.wait(timeout=0.5) + self._process.wait(timeout=timeout_seconds) except subprocess.TimeoutExpired: + timed_out = True self._process.terminate() try: self._process.wait(timeout=0.5) @@ -275,6 +284,7 @@ def _shutdown(self) -> None: stream.close() except OSError: pass + return timed_out class ExternalBookReplayAdapterFactory: diff --git a/src/tracebook/conformance/external.py b/src/tracebook/conformance/external.py index e4f1d48..c33b668 100644 --- a/src/tracebook/conformance/external.py +++ b/src/tracebook/conformance/external.py @@ -253,19 +253,28 @@ def close(self) -> None: ) finally: self._closed = True - self._shutdown() - if not was_broken and close_error is None and self._process.returncode != 0: - close_error = AdapterProtocolError( - f"candidate exited with code {self._process.returncode} after complete" + shutdown_timed_out = self._shutdown( + timeout_seconds=self.timeout_seconds if not self._broken else 0.5 ) + if not was_broken and close_error is None: + if shutdown_timed_out: + close_error = AdapterProtocolError( + f"candidate timed out exiting after complete ({self.timeout_seconds:g}s)" + ) + elif self._process.returncode != 0: + close_error = AdapterProtocolError( + f"candidate exited with code {self._process.returncode} after complete" + ) if close_error is not None: raise close_error - def _shutdown(self) -> None: + def _shutdown(self, *, timeout_seconds: float = 0.5) -> bool: + timed_out = False if self._process.poll() is None: try: - self._process.wait(timeout=0.5) + self._process.wait(timeout=timeout_seconds) except subprocess.TimeoutExpired: + timed_out = True self._process.terminate() try: self._process.wait(timeout=0.5) @@ -285,6 +294,7 @@ def _shutdown(self) -> None: stream.close() except OSError: pass + return timed_out class ExternalProcessAdapterFactory: diff --git a/src/tracebook/visualization/dashboard.py b/src/tracebook/visualization/dashboard.py index 2d1d46a..2daac08 100644 --- a/src/tracebook/visualization/dashboard.py +++ b/src/tracebook/visualization/dashboard.py @@ -193,7 +193,7 @@ def set_order_book_manager(self, manager: OrderBookManager): def _create_app(self) -> dash.Dash: """Create the Dash application.""" - app = dash.Dash(__name__) + app = dash.Dash(__name__, title="Tracebook · Performance Dashboard") app.layout = html.Div( # type: ignore[misc] # Dash types layout as read-only [ diff --git a/tests/test_book_replay.py b/tests/test_book_replay.py index eaf79e0..fbcf208 100644 --- a/tests/test_book_replay.py +++ b/tests/test_book_replay.py @@ -13,6 +13,7 @@ BookReplayError, BookReplayEvent, BookReplayObservation, + BookReplayProtocolError, BookReplaySnapshot, BookReplayState, EngineMetadata, @@ -230,7 +231,7 @@ def test_external_example_adapter_conforms(): lambda config: ExternalBookReplayAdapter( [sys.executable, str(EXAMPLE_ADAPTER)], config, - timeout_seconds=2, + timeout_seconds=10, ), ) @@ -238,6 +239,69 @@ def test_external_example_adapter_conforms(): assert report.candidate_engine.name == "example-book-replay-adapter" +def _shutdown_test_adapter(delay, exit_code, *, handle_terminate=False): + script = """ +import json +import signal +import sys +import time + +delay, exit_code, handle_terminate = json.loads(sys.argv[1]) +if handle_terminate: + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) +hello = json.loads(sys.stdin.readline()) +print(json.dumps({ + "type": "ready", + "protocol": hello["protocol"], + "protocol_version": hello["protocol_version"], + "engine": {"name": "shutdown-test", "version": "1", "language": "Python"}, +}), flush=True) +finish = json.loads(sys.stdin.readline()) +print(json.dumps({"type": "complete", "event_count": finish["event_count"]}), flush=True) +time.sleep(delay) +sys.exit(exit_code) +""" + return ExternalBookReplayAdapter( + [ + sys.executable, + "-I", + "-u", + "-c", + script, + json.dumps([delay, exit_code, handle_terminate]), + ], + BookReplayConfig(), + timeout_seconds=10, + ) + + +def test_external_close_allows_configured_time_for_clean_exit(): + adapter = _shutdown_test_adapter(0.75, 0) + + adapter.close() + adapter.close() + + assert adapter._process.returncode == 0 + + +def test_external_close_rejects_nonzero_exit_after_complete(): + adapter = _shutdown_test_adapter(0, 7) + + with pytest.raises(BookReplayProtocolError, match="exited with code 7 after complete"): + adapter.close() + + +def test_external_close_rejects_timeout_even_when_termination_exits_zero(): + adapter = _shutdown_test_adapter(60, 0, handle_terminate=True) + # Startup has its own generous budget; now isolate the exit timeout. + adapter.timeout_seconds = 0.2 + + with pytest.raises(BookReplayProtocolError, match="timed out exiting after complete"): + adapter.close() + + assert adapter._process.returncode is not None + + def test_cli_sample_and_external_run(tmp_path, capsys): sample_dir = tmp_path / "sample" assert main(["sample", str(sample_dir)]) == 0 diff --git a/tests/test_conformance_exit_codes.py b/tests/test_conformance_exit_codes.py index d6bf906..3466b49 100644 --- a/tests/test_conformance_exit_codes.py +++ b/tests/test_conformance_exit_codes.py @@ -301,7 +301,7 @@ def test_cli_returns_two_for_malformed_mid_run_stdout(tmp_path): assert "not valid JSON" in payload["divergence"]["message"] -def test_cli_returns_two_for_mid_run_timeout(tmp_path): +def test_cli_returns_two_for_mid_run_timeout(tmp_path, monkeypatch): events = tmp_path / "events.jsonl" report_path = tmp_path / "report.json" adapter = tmp_path / "timeout_adapter.py" @@ -325,6 +325,16 @@ def test_cli_returns_two_for_mid_run_timeout(tmp_path): encoding="utf-8", ) + class MidRunTimeoutFactory(conformance_cli.ExternalProcessAdapterFactory): + def __call__(self, config): + candidate = super().__call__(config) + # Exercise the response timeout after ready, not interpreter startup + # on a busy host. Startup failures intentionally have no run report. + candidate.timeout_seconds = 0.2 + return candidate + + monkeypatch.setattr(conformance_cli, "ExternalProcessAdapterFactory", MidRunTimeoutFactory) + exit_code = main( [ "run", @@ -332,7 +342,7 @@ def test_cli_returns_two_for_mid_run_timeout(tmp_path): "--output", str(report_path), "--timeout", - "0.2", + "10", "--candidate", sys.executable, str(adapter), @@ -342,7 +352,7 @@ def test_cli_returns_two_for_mid_run_timeout(tmp_path): payload = json.loads(report_path.read_text(encoding="utf-8")) assert exit_code == 2 assert payload["divergence"]["category"] == "protocol" - assert "timed out" in payload["divergence"]["message"] + assert "timed out waiting for 'observation'" in payload["divergence"]["message"] @pytest.mark.parametrize( diff --git a/tests/test_conformance_shutdown.py b/tests/test_conformance_shutdown.py new file mode 100644 index 0000000..cb611c0 --- /dev/null +++ b/tests/test_conformance_shutdown.py @@ -0,0 +1,122 @@ +"""A complete frame is valid evidence only when the peer exits cleanly in time.""" + +import json +import os +import sys +import time + +import pytest + +from tracebook.conformance import ( + AdapterProtocolError, + ConformanceConfig, + ExternalProcessAdapter, + run_conformance, +) +from tracebook.conformance.exit_codes import exit_code_for_artifact + + +def _shutdown_peer(delay, exit_code, *, handle_terminate=False, broken_snapshot=False): + script = """ +import json +import signal +import sys +import time + +delay, exit_code, handle_terminate, broken_snapshot = json.loads(sys.argv[1]) +if handle_terminate: + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) +hello = json.loads(sys.stdin.readline()) +print(json.dumps({ + "type": "ready", + "protocol": hello["protocol"], + "protocol_version": hello["protocol_version"], + "engine": {"name": "shutdown-test", "version": "1", "language": "Python"}, +}), flush=True) +for line in sys.stdin: + message = json.loads(line) + if message["type"] == "snapshot": + if broken_snapshot: + print("broken snapshot", flush=True) + time.sleep(60) + else: + print(json.dumps({"type": "snapshot", "index": 0, "state": {"books": []}}), flush=True) + elif message["type"] == "finish": + print(json.dumps({"type": "complete", "event_count": message["event_count"]}), flush=True) + time.sleep(delay) + sys.exit(exit_code) +""" + return ExternalProcessAdapter( + [ + sys.executable, + "-I", + "-u", + "-c", + script, + json.dumps([delay, exit_code, handle_terminate, broken_snapshot]), + ], + ConformanceConfig(), + timeout_seconds=10, + ) + + +def test_close_allows_configured_time_for_clean_exit(): + adapter = _shutdown_peer(0.75, 0) + + adapter.close() + adapter.close() + + assert adapter._process.returncode == 0 + + +def test_close_rejects_nonzero_exit_after_complete(): + adapter = _shutdown_peer(0, 7) + + with pytest.raises(AdapterProtocolError, match="exited with code 7 after complete"): + adapter.close() + + assert adapter._process.returncode == 7 + + +@pytest.mark.skipif(os.name == "nt", reason="requires a catchable POSIX SIGTERM") +def test_close_rejects_timeout_even_when_termination_exits_zero(): + adapter = _shutdown_peer(60, 0, handle_terminate=True) + # Give startup its own generous budget before testing just the exit bound. + adapter.timeout_seconds = 0.5 + + with pytest.raises(AdapterProtocolError, match="timed out exiting after complete"): + adapter.close() + + assert adapter._process.returncode == 0 + adapter.close() + + +def test_forced_shutdown_invalidates_otherwise_conformant_report(): + adapter = _shutdown_peer(60, 0, handle_terminate=True) + adapter.timeout_seconds = 0.5 + + report = run_conformance([], lambda config: adapter) + + assert report.conformant is False + assert report.divergence.kind == "adapter_close_error" + assert "timed out exiting after complete" in report.divergence.message + assert report.operational_failure is True + assert exit_code_for_artifact(report.to_dict()) == 2 + assert adapter._process.returncode is not None + + +def test_broken_stream_keeps_original_error_and_uses_prompt_cleanup(): + adapter = _shutdown_peer(60, 0, handle_terminate=True, broken_snapshot=True) + started = time.monotonic() + + report = run_conformance([], lambda config: adapter) + + assert report.conformant is False + assert report.divergence.kind == "adapter_error" + assert "not valid JSON" in report.divergence.message + assert report.divergence.close_error is None + assert report.operational_failure is True + assert exit_code_for_artifact(report.to_dict()) == 2 + assert adapter._process.returncode is not None + # A broken stream should not consume the healthy peer's ten-second budget. + assert time.monotonic() - started < 5 diff --git a/tests/test_dashboard_demo_construction.py b/tests/test_dashboard_demo_construction.py index 7edf45e..52c2732 100644 --- a/tests/test_dashboard_demo_construction.py +++ b/tests/test_dashboard_demo_construction.py @@ -29,8 +29,12 @@ def __getattr__(self, name): return lambda *args, **kwargs: (name, args, kwargs) class FakeDash: + last_args = None + last_kwargs = None + def __init__(self, *args, **kwargs): - pass + FakeDash.last_args = args + FakeDash.last_kwargs = kwargs def callback(self, *args, **kwargs): return lambda func: func @@ -54,6 +58,20 @@ def callback(self, *args, **kwargs): monkeypatch.setitem(sys.modules, "pandas", pandas_module) +def test_dashboard_sets_tracebook_browser_title(monkeypatch): + _install_dashboard_dependency_stubs(monkeypatch) + sys.modules.pop("tracebook.visualization.dashboard", None) + dashboard_module = importlib.import_module("tracebook.visualization.dashboard") + + try: + dashboard_module.PerformanceDashboard() + assert dashboard_module.dash.Dash.last_kwargs["title"] == ( + "Tracebook · Performance Dashboard" + ) + finally: + sys.modules.pop("tracebook.visualization.dashboard", None) + + def test_dashboard_demo_mode_constructs_engine_and_dashboard_without_running_server( monkeypatch, ): diff --git a/tests/test_distribution_privacy.py b/tests/test_distribution_privacy.py new file mode 100644 index 0000000..509578e --- /dev/null +++ b/tests/test_distribution_privacy.py @@ -0,0 +1,179 @@ +"""Release archives must not inherit private files or stale source inventories.""" + +from __future__ import annotations + +import io +import os +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from tools.verify_distribution_privacy import PrivacyError, main, verify_artifact + +ROOT = Path(__file__).resolve().parents[1] +PRIVATE_PATHS = ( + "experiments/private/agent-qualification-v2/gold.json", + "experiments/private/agent-qualification-v2/execution/controller.py", + ".agents/skills/local/SKILL.md", + ".codex/settings.json", + "openwiki/quickstart.md", + "graphify-out/graph.json", + ".local-tools/helper.py", + "docs/openwiki.md", + "docs/graphify.md", + "AGENTS.md", + "CLAUDE.md", + "skills-lock.json", + "examples/AGENTS.md", +) + + +def _write_archive(path: Path, files: dict[str, bytes]) -> None: + if path.name.endswith(".whl"): + with zipfile.ZipFile(path, "w") as archive: + for name, data in files.items(): + archive.writestr(name, data) + else: + with tarfile.open(path, "w:gz") as archive: + for name, data in files.items(): + member = tarfile.TarInfo(f"privacy_fixture-1.0/{name}") + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + + +@pytest.mark.parametrize("suffix", (".whl", ".tar.gz")) +@pytest.mark.parametrize( + "private_path", + ( + *PRIVATE_PATHS, + "tracebook.data/data/experiments/private/gold.json", + "src/tracebook/.agents/x.py", + ), +) +def test_privacy_gate_rejects_private_archive_members(tmp_path, suffix, private_path): + artifact = tmp_path / f"contaminated{suffix}" + _write_archive(artifact, {private_path: b"synthetic private sentinel\n"}) + + with pytest.raises(PrivacyError, match="private path"): + verify_artifact(artifact) + + +@pytest.mark.parametrize("suffix", (".whl", ".tar.gz")) +def test_privacy_gate_rejects_stale_sources_without_private_members(tmp_path, suffix): + artifact = tmp_path / f"stale-inventory{suffix}" + _write_archive( + artifact, + { + "pyproject.toml": b"# synthetic public file\n", + "src/privacy_fixture.egg-info/SOURCES.txt": ( + b"pyproject.toml\nexperiments/private/synthetic-gold.json\n" + ), + }, + ) + + with pytest.raises(PrivacyError, match="SOURCES.txt.*experiments/private/synthetic-gold.json"): + verify_artifact(artifact) + + +def test_privacy_gate_cli_checks_every_artifact_and_fails_closed(tmp_path, capsys): + clean = tmp_path / "clean.whl" + leaked = tmp_path / "leaked.tar.gz" + _write_archive(clean, {"tracebook/__init__.py": b"# synthetic public source\n"}) + _write_archive(leaked, {"experiments/private/gold.json": b"{}\n"}) + + assert main([str(clean), str(leaked)]) == 1 + assert "leaked.tar.gz" in capsys.readouterr().err + assert main([str(clean)]) == 0 + assert "verified for 1 artifacts" in capsys.readouterr().out + assert main([str(tmp_path / "missing.whl")]) == 1 + + +def test_real_manifest_excludes_synthetic_private_tree_and_stale_cache(tmp_path): + """Build only synthetic files with the real manifest, never the private tree.""" + + project = tmp_path / "source" + project.mkdir() + (project / "pyproject.toml").write_text( + """\ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "privacy-fixture" +version = "1.0" +readme = "README.md" + +[tool.setuptools] +package-dir = { "" = "src" } +include-package-data = true + +[tool.setuptools.packages.find] +where = ["src"] +""", + encoding="utf-8", + ) + (project / "MANIFEST.in").write_text( + (ROOT / "MANIFEST.in").read_text(encoding="utf-8"), encoding="utf-8" + ) + public_files = { + "README.md": "Synthetic build regression.\n", + "LICENSE": "MIT\n", + "src/tracebook/__init__.py": "# synthetic public package\n", + "src/tracebook/py.typed": "", + "experiments/public-evidence.json": '{"synthetic": true}\n', + "experiments/public_runner.py": "# synthetic public runner\n", + "integrations/intrepid_orderbook/main.go": "package main\n", + "integrations/intrepid_orderbook/go.mod": "module privacy-fixture\n", + "integrations/intrepid_orderbook/go.sum": "synthetic public lockfile\n", + } + for path, content in public_files.items(): + target = project / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + for path in PRIVATE_PATHS: + target = project / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("synthetic private sentinel\n", encoding="utf-8") + + # setuptools reads an existing inventory before applying MANIFEST.in. + # Seed paths that ordinary recursive includes would not otherwise discover. + cache = project / "src/privacy_fixture.egg-info/SOURCES.txt" + cache.parent.mkdir(parents=True) + cache.write_text("\n".join((*public_files, *PRIVATE_PATHS)) + "\n", encoding="utf-8") + environment = os.environ.copy() + environment["PIP_NO_INDEX"] = "1" + result = subprocess.run( + [ + sys.executable, + "-m", + "build", + "--sdist", + "--wheel", + "--no-isolation", + "--outdir", + str(tmp_path / "dist"), + str(project), + ], + env=environment, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + artifacts = sorted((tmp_path / "dist").iterdir()) + assert len(artifacts) == 2 + for artifact in artifacts: + verify_artifact(artifact) + + sdist = next(path for path in artifacts if path.name.endswith(".tar.gz")) + with tarfile.open(sdist, "r:gz") as archive: + members = {name.partition("/")[2] for name in archive.getnames()} + assert set(public_files) <= members + assert not set(PRIVATE_PATHS) & members + cached_paths = set(cache.read_text(encoding="utf-8").splitlines()) + assert not set(PRIVATE_PATHS) & cached_paths + assert all((project / path).is_file() for path in PRIVATE_PATHS) diff --git a/tests/test_external_integrations.py b/tests/test_external_integrations.py index 124edaf..e099202 100644 --- a/tests/test_external_integrations.py +++ b/tests/test_external_integrations.py @@ -86,6 +86,9 @@ def test_nautilus_book_replay_integration_is_version_and_source_pinned(): assert "LGPL-3.0-only" in readme assert "does not claim" in readme assert '"nautilus-trader==2.0.0rc3"' in workflow + assert "actions/setup-python@v7" in workflow + assert 'python -m pip install -e . "nautilus-trader==2.0.0rc3" "pytest>=9.1.1"' in workflow + assert "python -m pytest" in workflow assert "9e0af6be935dce940a87497788c7a9a799c71f05e34a0204c9d294fce611b002" in workflow assert "sha256:09b1599eaeb474d98617acc7869ace26759ed5ab8350803f06197ef572864bab" in workflow assert "failure-dfe5c23848b63211b655" in readme @@ -212,9 +215,14 @@ def test_intrepid_orderbook_is_pinned_qualified_and_retains_the_fok_boundary(): assert "github.com/intrepidkarthi/orderbook v0.26.0" in go_mod assert "github.com/intrepidkarthi/orderbook v0.26.0 h1:" in go_sum + assert "actions/setup-python@v7" in workflow assert "51d480cdb68b9989febb0b075d291cf891f425b3" in readme assert "submitted-order versus submitted-order" in readme - assert "at most six non-zero quantity decimal places" in readme + assert "Inputs requiring more than six" in readme + assert "non-zero decimal places are explicitly rejected" in readme + assert "rounds output quantities only" in readme + assert "maps to a distinct native account" in readme + assert "incoming quantity times each potentially crossed maker's" in readme assert "6/9" in readme assert "failure-1ce2954857800b3a068d" in readme assert "sha256:f8db775baabe7e665d45bbb920a67e43d405acf445de4951617df68ffa24eb69" in workflow diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index 2b93d67..fb4b6b8 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -185,7 +185,7 @@ def test_release_publishes_and_verifies_the_source_owner_before_the_facade(): ) -def test_sdist_excludes_local_navigation_material(): +def test_sdist_excludes_private_experiments_and_local_navigation_material(): manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8") for directive in ( @@ -196,6 +196,10 @@ def test_sdist_excludes_local_navigation_material(): "prune openwiki", "prune graphify-out", "prune .local-tools", + "prune experiments/private", + "prune .agents", + "prune .codex", + "global-exclude AGENTS.md CLAUDE.md skills-lock.json", "recursive-include packaging *.md", "include packaging/tracebook-sim/pyproject.toml", "include packaging/tracebook-sim/LICENSE", @@ -203,6 +207,12 @@ def test_sdist_excludes_local_navigation_material(): ): assert directive in manifest + for path in ("Makefile", ".github/workflows/ci.yml", ".github/workflows/release.yml"): + workflow = (ROOT / path).read_text(encoding="utf-8") + assert ( + "tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*" in workflow + ) + def test_public_install_surfaces_warn_before_the_legacy_ownership_handoff(): install_surfaces = ( diff --git a/tools/verify_distribution_privacy.py b/tools/verify_distribution_privacy.py new file mode 100644 index 0000000..79694dd --- /dev/null +++ b/tools/verify_distribution_privacy.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Reject private experiment and local navigation paths in release artifacts. + +Checks archive member names and setuptools' SOURCES.txt inventories without +extracting artifacts. The latter prevents publishing stale local path metadata +even when the referenced private files are absent from the archive. +""" + +from __future__ import annotations + +import argparse +import sys +import tarfile +import zipfile +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Sequence + +PRIVATE_ROOTS = ( + "experiments/private", + ".agents", + ".codex", + "openwiki", + "graphify-out", + ".local-tools", +) +PRIVATE_FILES = {"docs/openwiki.md", "docs/graphify.md"} +LOCAL_POINTER_NAMES = {"AGENTS.md", "CLAUDE.md", "skills-lock.json"} + + +class PrivacyError(ValueError): + """A release artifact contains private paths or cannot be checked.""" + + +def _check_path(path: str, *, context: str) -> None: + normalized = path.rstrip("/") + parts = normalized.split("/") + if ( + not normalized + or "\\" in normalized + or "\x00" in normalized + or any(part in {"", ".", ".."} for part in parts) + or PureWindowsPath(normalized).drive + ): + raise PrivacyError(f"unsafe path in {context}: {path!r}") + if ( + any(normalized == path or normalized.endswith("/" + path) for path in PRIVATE_FILES) + or PurePosixPath(normalized).name in LOCAL_POINTER_NAMES + or any( + parts[index : index + len(root.split("/"))] == root.split("/") + for root in PRIVATE_ROOTS + for index in range(len(parts)) + ) + ): + raise PrivacyError(f"private path in {context}: {normalized}") + + +def _check_source_inventory(data: bytes, *, context: str) -> None: + try: + inventory = data.decode("utf-8") + except UnicodeDecodeError as error: + raise PrivacyError(f"cannot decode source inventory in {context}") from error + for path in inventory.splitlines(): + if path: + _check_path(path, context=context) + + +def verify_artifact(artifact: Path) -> None: + """Check one built wheel or gzip source distribution, without extraction.""" + + if artifact.name.endswith(".whl"): + with zipfile.ZipFile(artifact) as wheel: + for member in wheel.infolist(): + _check_path(member.filename, context=artifact.name) + if PurePosixPath(member.filename).name == "SOURCES.txt" and not member.is_dir(): + _check_source_inventory( + wheel.read(member), context=f"{artifact.name}:{member.filename}" + ) + elif artifact.name.endswith(".tar.gz"): + with tarfile.open(artifact, "r:gz") as archive: + roots = set() + for source_member in archive: + _check_path(source_member.name, context=artifact.name) + root, separator, relative = source_member.name.rstrip("/").partition("/") + roots.add(root) + if not separator: + if not source_member.isdir(): + raise PrivacyError( + f"sdist member has no project root: {source_member.name!r}" + ) + continue + _check_path(relative, context=artifact.name) + if PurePosixPath(relative).name == "SOURCES.txt": + source = archive.extractfile(source_member) if source_member.isreg() else None + if source is None: + raise PrivacyError( + f"cannot inspect source inventory: {source_member.name!r}" + ) + with source: + _check_source_inventory( + source.read(), context=f"{artifact.name}:{relative}" + ) + if len(roots) != 1: + raise PrivacyError(f"sdist must contain one project root: {artifact.name}") + else: + raise PrivacyError(f"expected a wheel or .tar.gz sdist: {artifact}") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifacts", type=Path, nargs="+") + args = parser.parse_args(argv) + try: + for artifact in args.artifacts: + verify_artifact(artifact) + except (PrivacyError, OSError, tarfile.TarError, zipfile.BadZipFile) as error: + print(f"distribution privacy check failed: {error}", file=sys.stderr) + return 1 + print(f"Distribution privacy verified for {len(args.artifacts)} artifacts.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 0c98e182fb77758782d08999b920c5826f8090be Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Fri, 4 Sep 2026 21:40:23 -0400 Subject: [PATCH 15/19] Harden native quantity boundaries and evidence publication --- .github/workflows/gocronx-matcher.yml | 8 +- .github/workflows/intrepid-orderbook.yml | 8 +- .../workflows/nautilus-trader-book-replay.yml | 14 + .github/workflows/orderbook-rs.yml | 8 +- .github/workflows/python-matching-engine.yml | 8 +- CHANGELOG.md | 12 + CONTRIBUTING.md | 8 +- Makefile | 2 +- docs/book-replay-conformance.md | 19 + docs/independent-onboarding.md | 14 +- docs/release.md | 6 +- integrations/gocronx_matcher/README.md | 13 +- integrations/gocronx_matcher/src/adapter.rs | 122 ++++-- integrations/orderbook_rs/README.md | 16 +- integrations/orderbook_rs/src/adapter.rs | 134 ++++-- integrations/rust_protocol/README.md | 20 + integrations/rust_protocol/src/lib.rs | 2 + integrations/rust_protocol/src/quantity.rs | 179 ++++++++ src/tracebook/book_replay/artifacts.py | 204 +++++++++ src/tracebook/book_replay/cli.py | 124 +++--- tests/test_book_replay_artifacts.py | 406 ++++++++++++++++++ tests/test_conformance.py | 2 +- tests/test_integration_workflow_coverage.py | 134 ++++++ tests/test_release_metadata.py | 15 +- tests/test_release_virtualenvs.py | 47 ++ tools/smoke_conformance_wheel.py | 4 +- tools/verify_distribution_split.py | 4 +- 27 files changed, 1385 insertions(+), 148 deletions(-) create mode 100644 integrations/rust_protocol/src/quantity.rs create mode 100644 src/tracebook/book_replay/artifacts.py create mode 100644 tests/test_book_replay_artifacts.py create mode 100644 tests/test_integration_workflow_coverage.py create mode 100644 tests/test_release_virtualenvs.py diff --git a/.github/workflows/gocronx-matcher.yml b/.github/workflows/gocronx-matcher.yml index e6b7e56..c3b6d96 100644 --- a/.github/workflows/gocronx-matcher.yml +++ b/.github/workflows/gocronx-matcher.yml @@ -7,11 +7,17 @@ on: pull_request: paths: - ".github/workflows/gocronx-matcher.yml" + - "pyproject.toml" + - "MANIFEST.in" - "integrations/gocronx_matcher/**" - "integrations/rust_protocol/**" + - "src/tracebook/__init__.py" + - "src/tracebook/_version.py" + - "src/tracebook/core/**" - "src/tracebook/conformance/**" - - "src/tracebook/events/market_replay.py" + - "src/tracebook/events/**" - "tests/test_external_integrations.py" + - "tests/test_integration_workflow_coverage.py" permissions: contents: read diff --git a/.github/workflows/intrepid-orderbook.yml b/.github/workflows/intrepid-orderbook.yml index b9c9ad7..4baaa3e 100644 --- a/.github/workflows/intrepid-orderbook.yml +++ b/.github/workflows/intrepid-orderbook.yml @@ -7,10 +7,16 @@ on: pull_request: paths: - ".github/workflows/intrepid-orderbook.yml" + - "pyproject.toml" + - "MANIFEST.in" - "integrations/intrepid_orderbook/**" + - "src/tracebook/__init__.py" + - "src/tracebook/_version.py" + - "src/tracebook/core/**" - "src/tracebook/conformance/**" - - "src/tracebook/events/market_replay.py" + - "src/tracebook/events/**" - "tests/test_external_integrations.py" + - "tests/test_integration_workflow_coverage.py" permissions: contents: read diff --git a/.github/workflows/nautilus-trader-book-replay.yml b/.github/workflows/nautilus-trader-book-replay.yml index 70c0a91..e2a6c45 100644 --- a/.github/workflows/nautilus-trader-book-replay.yml +++ b/.github/workflows/nautilus-trader-book-replay.yml @@ -7,11 +7,24 @@ on: pull_request: paths: - ".github/workflows/nautilus-trader-book-replay.yml" + - "pyproject.toml" + - "MANIFEST.in" + - "integrations/__init__.py" - "integrations/nautilus_trader/**" + - "integrations/python_matching_engine/__init__.py" + - "integrations/python_matching_engine/adapter.py" + - "src/tracebook/__init__.py" + - "src/tracebook/_version.py" + - "src/tracebook/core/**" + - "src/tracebook/conformance/**" + - "src/tracebook/events/**" - "src/tracebook/book_replay/**" + - "examples/book_replay_adapter.py" + - "tests/conftest.py" - "tests/test_book_replay*.py" - "tests/fixtures/faulty_book_replay_adapter.py" - "tests/test_external_integrations.py" + - "tests/test_integration_workflow_coverage.py" permissions: contents: read @@ -118,6 +131,7 @@ jobs: python -m pytest tests/test_book_replay.py tests/test_book_replay_campaign.py + tests/test_book_replay_artifacts.py tests/test_external_integrations.py - uses: actions/upload-artifact@v7 diff --git a/.github/workflows/orderbook-rs.yml b/.github/workflows/orderbook-rs.yml index 0cd8605..362f20c 100644 --- a/.github/workflows/orderbook-rs.yml +++ b/.github/workflows/orderbook-rs.yml @@ -7,13 +7,19 @@ on: pull_request: paths: - ".github/workflows/orderbook-rs.yml" + - "pyproject.toml" + - "MANIFEST.in" - "integrations/flash_benchmark/**" - "integrations/orderbook_rs/**" - "integrations/rust_protocol/**" + - "src/tracebook/__init__.py" + - "src/tracebook/_version.py" + - "src/tracebook/core/**" - "src/tracebook/conformance/**" - - "src/tracebook/events/market_replay.py" + - "src/tracebook/events/**" - "tests/test_external_integrations.py" - "tests/test_flash_bridge.py" + - "tests/test_integration_workflow_coverage.py" permissions: contents: read diff --git a/.github/workflows/python-matching-engine.yml b/.github/workflows/python-matching-engine.yml index b0b92cf..ad13f28 100644 --- a/.github/workflows/python-matching-engine.yml +++ b/.github/workflows/python-matching-engine.yml @@ -7,11 +7,17 @@ on: pull_request: paths: - ".github/workflows/python-matching-engine.yml" + - "pyproject.toml" + - "MANIFEST.in" - "integrations/python_matching_engine/**" + - "src/tracebook/__init__.py" + - "src/tracebook/_version.py" + - "src/tracebook/core/**" - "src/tracebook/conformance/**" - - "src/tracebook/events/market_replay.py" + - "src/tracebook/events/**" - "tests/test_conformance_campaign.py" - "tests/test_external_integrations.py" + - "tests/test_integration_workflow_coverage.py" permissions: contents: read diff --git a/CHANGELOG.md b/CHANGELOG.md index d8ac5e5..92d1ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions. ## Unreleased +- Kept Rust adapter input quantities independent of report display precision. + Both native adapters now share exact checked 12-place integer encoding and + reject quantities outside that domain rather than pre-rounding orders. +- Made L3 report and reduced-trace outputs no-clobber: reserve destinations + before candidate execution, stage complete artifacts, and publish the report + only after its reduced trace. Existing files and competing writes are kept. +- Expanded native integration CI path filters to cover reference-engine and + packaging changes. Local type checks now use the active Python minor version, + matching CI, and release instructions include distribution privacy checks. +- Aligned release-check virtual environments with the standard POSIX symlink + behavior, so relocatable Python installations can run clean-install and + migration checks without broken copied-executable library paths. - Added a separate L3 book-replay contract and pinned NautilusTrader adapter. This exercises market-data deltas and non-mutating FIFO fill probes, not a submitted-order CLOB. Both process clients now honor the configured shutdown diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c09bbc..97eead7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ Run these before opening a pull request: ```bash python -m black --check src tests examples integrations experiments tools python -m flake8 src tests examples integrations experiments tools -python -m mypy src/tracebook experiments tools +python -m mypy --python-version "$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" src/tracebook experiments tools python -m bandit -q -r src integrations tools python -m compileall -q src tests examples integrations experiments tools python -m pytest --cov=tracebook --cov-report=term-missing --cov-fail-under=75 @@ -37,6 +37,7 @@ For packaging changes, also run: ```bash python -m build --sdist --wheel --outdir dist/conformance . python -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim +python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/* python -m twine check dist/conformance/* dist/simulator/* python tools/verify_distribution_split.py \ --expected-version 0.6.0 \ @@ -52,6 +53,11 @@ python tools/verify_sdist_wheel_agreement.py \ python tools/smoke_conformance_wheel.py dist/conformance/*.whl ``` +Type-check against the active interpreter, as CI does for each supported Python +version. Newer optional dependencies can contain syntax that cannot be parsed +when a newer environment is accidentally checked as Python 3.10. Passing on one +interpreter does not replace the Python 3.10–3.13 CI matrix. + Normalized feed adapters should emit `tracebook.MarketEvent` values and keep venue-specific parsing, sequence checks, and assumptions outside the core replay engine. Include a small fixture and strict rejection tests with each adapter. diff --git a/Makefile b/Makefile index 2c32fc1..baf144f 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ lint: $(PYTHON_VENV) -m flake8 src/ tests/ examples/ integrations/ experiments/ tools/ typecheck: - $(PYTHON_VENV) -m mypy src/tracebook experiments tools + $(PYTHON_VENV) -m mypy --python-version "$$($(PYTHON_VENV) -c 'import sys; print("%d.%d" % sys.version_info[:2])')" src/tracebook experiments tools security: $(PYTHON_VENV) -m bandit -q -r src integrations tools diff --git a/docs/book-replay-conformance.md b/docs/book-replay-conformance.md index 6627737..5f07a2f 100644 --- a/docs/book-replay-conformance.md +++ b/docs/book-replay-conformance.md @@ -99,6 +99,25 @@ python -m tracebook.book_replay run \ Exit `0` means semantic agreement, `1` means the first semantic divergence was recorded, and `2` means input, process, or protocol failure. +File outputs must be new paths. Existing files, directories, and symlinks are +refused with exit `2`; input/output aliases and outputs nested inside one +another are also refused. All requested destinations are reserved before the +candidate starts, including a campaign's optional reduced trace even when the +campaign ultimately passes. A passing campaign leaves no reduced-trace file. + +Reports and traces are staged completely before atomic publication without +overwriting another writer's file. When producing a pair, the reduced trace is +published first and the report last; a caught publication failure rolls back +files created by that attempt. These are individual file commits, not a +filesystem-wide transaction: abrupt process termination can leave a complete +trace without its report. Only use the pair after the command completes. +Sibling `.NAME.tracebook-in-progress` reservations and `.tracebook-stage-*` +temporary files are removed on normal completion, with cleanup attempted on +handled errors. Filesystem cleanup failures or competing replacements can +leave sidecars. After a forced stop or cleanup failure, retain the artifacts +and use fresh output names for a new invocation; do not remove a live process's +reservations. + ## Generated Campaigns And Reduction The v1 generator uses a specified SplitMix64 stream rather than Python's diff --git a/docs/independent-onboarding.md b/docs/independent-onboarding.md index b662c5f..38f5497 100644 --- a/docs/independent-onboarding.md +++ b/docs/independent-onboarding.md @@ -130,7 +130,14 @@ is neither. Report the attempt through the existing Engine qualification issue form, adding the wheel hash, evidence manifest and human-correction log. Remove credentials, proprietary traces and machine-specific paths before publication. -## Proposed Maintainer Invitation (Not Sent) +## Maintainer Invitation + +Sent on 2026-09-05 (UTC) in +[intrepidkarthi/orderbook issue #25](https://github.com/intrepidkarthi/orderbook/issues/25), +with the packet pinned to verified Tracebook commit +`55e3c4b010c8cc1bc967db59f94f593ffa0f6838` and the local rehearsal's artifact +hashes. Sending the invitation is not independent use or a retention decision; +the measurement fields above remain unobserved. > We have a pinned, out-of-tree Tracebook adapter for your v0.26.0 CLOB. FIFO > limit qualification passes; the broader FOK profile stops on your native @@ -140,6 +147,5 @@ credentials, proprietary traces and machine-specific paths before publication. > change or in-tree adapter commitment is requested. We would record setup > time, corrections, and your retention decision, including a failed attempt. -Attach the reviewed Tracebook source revision and artifact hashes before -sending. A maintainer response and any independent result must be recorded as -new evidence; this prepared invitation establishes neither. +A maintainer response and any independent result must be recorded as new +evidence; the sent invitation establishes neither. diff --git a/docs/release.md b/docs/release.md index 66f5865..35d7bfb 100644 --- a/docs/release.md +++ b/docs/release.md @@ -21,7 +21,7 @@ python -m pip install -e ".[dev]" python -m pip install -e "./packaging/tracebook-sim[dashboard]" python -m black --check src tests examples integrations experiments tools python -m flake8 src tests examples integrations experiments tools -python -m mypy src/tracebook experiments tools +python -m mypy --python-version "$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" src/tracebook experiments tools python -m bandit -q -r src integrations tools python -m compileall -q src tests examples integrations experiments tools python -m pytest --cov=tracebook --cov-report=term-missing --cov-fail-under=75 @@ -33,6 +33,7 @@ tracebook-coinbase examples/data/coinbase_btcusd_l3_snapshot.json examples/data/ tracebook-corpus verify src/tracebook/corpus/fixtures/coinbase-btcusd-synthetic-v1 python -m build --sdist --wheel --outdir dist/conformance . python -m build --sdist --wheel --outdir dist/simulator packaging/tracebook-sim +python tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/* python -m twine check dist/conformance/* dist/simulator/* python -m pip download \ --no-deps \ @@ -69,6 +70,9 @@ python -m pip check - Confirm the native `orderbook-rs` integration passes its fixed trace, `7/9` suite profile, generated campaign, and intentional-drift negative control. - Confirm the pinned PythonMatchingEngine integration workflow passes. +- Confirm the pinned Go CLOB's FIFO qualifications and retained FOK divergence, + and the separate Nautilus L3 campaign and injected queue-priority control. + These are distinct contracts, not interchangeable qualification claims. - Check both generated package metadata sets and their disjoint wheel RECORDs. - Check that Dependabot has no urgent security update waiting. - Confirm no README badge or proof table overstates the current CI/test state. diff --git a/integrations/gocronx_matcher/README.md b/integrations/gocronx_matcher/README.md index 110da8a..8cc1ed1 100644 --- a/integrations/gocronx_matcher/README.md +++ b/integrations/gocronx_matcher/README.md @@ -47,8 +47,17 @@ traces, 10/10 covered capabilities, and 28 candidate process runs. | `clear` | replace the symbol's `OrderBook` | | full queue state | decode snapshot format version 1, then canonicalize bids descending and asks ascending | -The adapter scales prices into integer ticks and quantities into configured -fixed-point units. Source owners are retained in adapter metadata because +The adapter scales prices into integer ticks and quantities into fixed +twelve-place native lots. `quantity_decimal_places` controls output rounding +only; it never rounds submitted order, reduction, or replacement quantities. +For example, at output precision zero, crossing a `2.4` buy with a `3.5` sell +leaves native quantity `1.1`, reported as `1`. Nonrepresentable quantities are +rejected before submission, and invalid replacement quantities preserve the +existing order. See the shared crate's [numeric boundary](../rust_protocol/README.md) +for range and precision limits, including accepted configurations that remain +unqualified. + +Source owners are retained in adapter metadata because `matcher` does not store them, then reconciled against the native snapshot after every event. diff --git a/integrations/gocronx_matcher/src/adapter.rs b/integrations/gocronx_matcher/src/adapter.rs index f9a8861..86ec5e8 100644 --- a/integrations/gocronx_matcher/src/adapter.rs +++ b/integrations/gocronx_matcher/src/adapter.rs @@ -1,10 +1,9 @@ use crate::wire::{ - BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, RestingOrder, - TradeFill, + BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, QuantityEncoding, + RestingOrder, TradeFill, }; use matcher::{BookEvent, Order, OrderBook, OrderId, Quantity, RejectReason, Side, Trade}; -use rust_decimal::prelude::ToPrimitive; -use rust_decimal::{Decimal, RoundingStrategy}; +use rust_decimal::Decimal; use std::collections::{BTreeMap, BTreeSet}; use std::str::FromStr; @@ -18,8 +17,7 @@ const KIND_ICEBERG: u8 = 6; struct AdapterConfig { tick_size: Decimal, tick_size_f64: f64, - quantity_decimal_places: u32, - quantity_scale: u64, + quantity_encoding: QuantityEncoding, } impl AdapterConfig { @@ -30,9 +28,7 @@ impl AdapterConfig { if config.self_trade_policy != "NONE" { return Err("self_trade_policy must be NONE".to_string()); } - if config.quantity_decimal_places > 18 { - return Err("quantity_decimal_places must be between 0 and 18".to_string()); - } + let quantity_encoding = QuantityEncoding::new(config.quantity_decimal_places)?; let tick_size = Decimal::from_str(&config.tick_size) .map_err(|_| "tick_size must be a positive finite decimal".to_string())?; @@ -46,15 +42,10 @@ impl AdapterConfig { if !tick_size_f64.is_finite() || tick_size_f64 <= 0.0 { return Err("tick_size must be a positive finite decimal".to_string()); } - let quantity_scale = 10_u64 - .checked_pow(config.quantity_decimal_places) - .ok_or_else(|| "quantity scale exceeds u64".to_string())?; - Ok(Self { tick_size, tick_size_f64, - quantity_decimal_places: config.quantity_decimal_places, - quantity_scale, + quantity_encoding, }) } @@ -74,25 +65,7 @@ impl AdapterConfig { } fn quantity_units(&self, value: &serde_json::Number) -> Result { - let quantity = Decimal::from_str(&value.to_string()) - .map_err(|_| "quantity must be a finite decimal".to_string())?; - if quantity <= Decimal::ZERO { - return Err("quantity must be positive".to_string()); - } - let normalized = quantity.round_dp_with_strategy( - self.quantity_decimal_places, - RoundingStrategy::MidpointNearestEven, - ); - let scaled = normalized - .checked_mul(Decimal::from(self.quantity_scale)) - .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?; - let units = scaled - .to_u64() - .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?; - if units == 0 { - return Err("quantity rounds to zero at the configured precision".to_string()); - } - Ok(units) + self.quantity_encoding.units(value) } fn format_price(&self, ticks: u64) -> Result { @@ -103,7 +76,7 @@ impl AdapterConfig { } fn format_quantity(&self, units: u64) -> String { - canonical_decimal(Decimal::from(units) / Decimal::from(self.quantity_scale)) + self.quantity_encoding.format(units) } } @@ -615,6 +588,85 @@ mod tests { serde_json::from_value(value).unwrap() } + #[test] + fn output_precision_does_not_change_native_matching() { + let mut adapter = Adapter::new(ConfigWire { + quantity_decimal_places: 0, + ..ConfigWire::default() + }) + .unwrap(); + let values = [ + serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}), + serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":3.5}), + ]; + let mut last = None; + for (index, value) in values.into_iter().enumerate() { + let observation = adapter.apply(&event(value), index as u64 + 1).unwrap(); + assert_eq!(observation.outcome.status, "applied"); + last = Some(observation); + } + let observation = last.unwrap(); + assert_eq!(observation.trades.len(), 1); + assert_eq!(observation.trades[0].quantity, "2"); + assert_eq!( + observation.state_hash, + "443d82df2af0902942c8614bcce6dc8d763a24db85e81917cc4843a9f4726300" + ); + let state = adapter.snapshot().unwrap(); + assert_eq!(state.books[0].asks.len(), 1); + assert_eq!(state.books[0].asks[0].remaining_quantity, "1"); + } + + #[test] + fn output_precision_does_not_round_reduce_or_replace_inputs() { + let mut adapter = Adapter::new(ConfigWire { + quantity_decimal_places: 0, + ..ConfigWire::default() + }) + .unwrap(); + for (index, value) in [ + serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}), + serde_json::json!({"op":"reduce","symbol":"TEST","order_id":1,"quantity":0.4}), + serde_json::json!({"op":"replace","symbol":"TEST","order_id":1,"quantity":3.5}), + serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":1.4}), + ].into_iter().enumerate() { + let observation = adapter.apply(&event(value), index as u64 + 1).unwrap(); + assert_eq!(observation.outcome.status, "applied"); + } + let state = adapter.snapshot().unwrap(); + assert_eq!(state.books[0].bids.len(), 1); + assert_eq!(state.books[0].bids[0].remaining_quantity, "2"); + } + + #[test] + fn nonrepresentable_quantities_preserve_existing_state() { + let mut adapter = Adapter::new(ConfigWire::default()).unwrap(); + let first = event(serde_json::json!({ + "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4,"owner":9 + })); + let initial = adapter.apply(&first, 1).unwrap(); + for quantity in ["0.0000000000001", "18446745"] { + for op in ["new", "reduce", "replace"] { + let next = event(serde_json::json!({ + "op":op,"symbol":"TEST","order_id":if op == "new" {2} else {1}, + "side":"BUY","price":100,"quantity":serde_json::Number::from_str(quantity).unwrap() + })); + let observation = adapter.apply(&next, 2).unwrap(); + assert_eq!(observation.outcome.status, "rejected"); + assert_eq!( + observation.outcome.reason, + Some(if op == "replace" { + "INVALID_REPLACEMENT" + } else { + "INVALID_ORDER" + }) + ); + assert_eq!(observation.state_hash, initial.state_hash); + assert!(observation.trades.is_empty()); + } + } + } + #[test] fn snapshot_is_canonical_price_time_order() { let mut adapter = Adapter::new(ConfigWire::default()).unwrap(); diff --git a/integrations/orderbook_rs/README.md b/integrations/orderbook_rs/README.md index 4b52f58..323ecdd 100644 --- a/integrations/orderbook_rs/README.md +++ b/integrations/orderbook_rs/README.md @@ -30,9 +30,19 @@ flowchart LR ``` The shared [`rust_protocol`](../rust_protocol) crate owns framing, protocol -validation, and canonical SHA-256 state serialization. `adapter.rs` owns numeric -conversion, source IDs, owners, lifecycle operations, trade translation, and -complete queue snapshots. `orderbook-rs` performs all matching. +validation, canonical SHA-256 state serialization, and exact quantity encoding. +`adapter.rs` owns price conversion, source IDs, owners, lifecycle operations, +trade translation, and complete queue snapshots. `orderbook-rs` performs all +matching. + +Quantities use fixed twelve-place native lots, regardless of the configured +output precision. `quantity_decimal_places` rounds observations only: a `2.4` +buy crossed by a `3.5` sell at output precision zero leaves a native `1.1` +remainder, reported as `1`. It must not become a two-unit remainder through +input rounding. Nonrepresentable quantities are rejected before submission; +invalid replacement quantities preserve the existing order. See the shared +crate's [numeric boundary](../rust_protocol/README.md) for range and precision +limits, including accepted configurations that remain unqualified. ## Run The Proof diff --git a/integrations/orderbook_rs/src/adapter.rs b/integrations/orderbook_rs/src/adapter.rs index 078b736..b1b0c1e 100644 --- a/integrations/orderbook_rs/src/adapter.rs +++ b/integrations/orderbook_rs/src/adapter.rs @@ -1,6 +1,6 @@ use crate::wire::{ - BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, RestingOrder, - TradeFill, + BookSnapshot, BookState, ConfigWire, MarketEvent, ObservationFrame, Outcome, QuantityEncoding, + RestingOrder, TradeFill, }; #[cfg(all(feature = "current", feature = "historical-issue-88"))] compile_error!("select either current or historical-issue-88, not both"); @@ -19,8 +19,8 @@ use pricelevel::{ use pricelevel_issue_88::{ Hash32, Id, OrderType, OrderUpdate, PriceLevelSnapshot, Quantity, Side, TimeInForce, }; -use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; -use rust_decimal::{Decimal, RoundingStrategy}; +use rust_decimal::Decimal; +use rust_decimal::prelude::FromPrimitive; use std::collections::{BTreeMap, BTreeSet}; use std::str::FromStr; use std::sync::{Arc, Mutex}; @@ -43,8 +43,7 @@ pub enum FaultMode { struct AdapterConfig { tick_size: Decimal, tick_size_f64: f64, - quantity_decimal_places: u32, - quantity_scale: u64, + quantity_encoding: QuantityEncoding, stp_mode: STPMode, } @@ -53,9 +52,7 @@ impl AdapterConfig { if !matches!(config.matching_algorithm.as_str(), "fifo" | "pro_rata") { return Err("matching_algorithm must be 'fifo' or 'pro_rata'".to_string()); } - if config.quantity_decimal_places > 18 { - return Err("quantity_decimal_places must be between 0 and 18".to_string()); - } + let quantity_encoding = QuantityEncoding::new(config.quantity_decimal_places)?; let tick_size = Decimal::from_str(&config.tick_size) .map_err(|_| "tick_size must be a positive finite decimal".to_string())?; @@ -81,15 +78,10 @@ impl AdapterConfig { ); } }; - let quantity_scale = 10_u64 - .checked_pow(config.quantity_decimal_places) - .ok_or_else(|| "quantity scale exceeds u64".to_string())?; - Ok(Self { tick_size, tick_size_f64, - quantity_decimal_places: config.quantity_decimal_places, - quantity_scale, + quantity_encoding, stp_mode, }) } @@ -110,25 +102,7 @@ impl AdapterConfig { } fn quantity_units(&self, value: &serde_json::Number) -> Result { - let quantity = Decimal::from_str(&value.to_string()) - .map_err(|_| "quantity must be a finite decimal".to_string())?; - if quantity <= Decimal::ZERO { - return Err("quantity must be positive".to_string()); - } - let normalized = quantity.round_dp_with_strategy( - self.quantity_decimal_places, - RoundingStrategy::MidpointNearestEven, - ); - let scaled = normalized - .checked_mul(Decimal::from(self.quantity_scale)) - .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?; - let units = scaled - .to_u64() - .ok_or_else(|| "quantity exceeds the adapter's integer range".to_string())?; - if units == 0 { - return Err("quantity rounds to zero at the configured precision".to_string()); - } - Ok(units) + self.quantity_encoding.units(value) } fn format_price(&self, ticks: u128) -> Result { @@ -141,8 +115,7 @@ impl AdapterConfig { } fn format_quantity(&self, units: u64) -> String { - let quantity = Decimal::from(units) / Decimal::from(self.quantity_scale); - canonical_decimal(quantity) + self.quantity_encoding.format(units) } } @@ -800,12 +773,95 @@ mod tests { } #[test] - fn quantity_conversion_uses_half_even_rounding() { + fn quantity_conversion_rounds_only_observations() { let config = config(); let low = serde_json::Number::from_str("1.2345").unwrap(); let high = serde_json::Number::from_str("1.2355").unwrap(); - assert_eq!(config.quantity_units(&low).unwrap(), 1_234); - assert_eq!(config.quantity_units(&high).unwrap(), 1_236); + assert_eq!(config.quantity_units(&low).unwrap(), 1_234_500_000_000); + assert_eq!(config.quantity_units(&high).unwrap(), 1_235_500_000_000); + assert_eq!(config.format_quantity(1_234_500_000_000), "1.234"); + assert_eq!(config.format_quantity(1_235_500_000_000), "1.236"); + } + + #[test] + fn output_precision_does_not_change_native_matching() { + let mut adapter = Adapter::new(ConfigWire { + quantity_decimal_places: 0, + ..ConfigWire::default() + }) + .unwrap(); + let values = [ + serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}), + serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":3.5}), + ]; + let mut last = None; + for (index, value) in values.into_iter().enumerate() { + let event = serde_json::from_value::(value).unwrap(); + let observation = adapter.apply(&event, index as u64 + 1).unwrap(); + assert_eq!(observation.outcome.status, "applied"); + last = Some(observation); + } + let observation = last.unwrap(); + assert_eq!(observation.trades.len(), 1); + assert_eq!(observation.trades[0].quantity, "2"); + assert_eq!( + observation.state_hash, + "443d82df2af0902942c8614bcce6dc8d763a24db85e81917cc4843a9f4726300" + ); + let state = adapter.snapshot().unwrap(); + assert_eq!(state.books[0].asks.len(), 1); + assert_eq!(state.books[0].asks[0].remaining_quantity, "1"); + } + + #[test] + fn output_precision_does_not_round_reduce_or_replace_inputs() { + let mut adapter = Adapter::new(ConfigWire { + quantity_decimal_places: 0, + ..ConfigWire::default() + }) + .unwrap(); + for (index, value) in [ + serde_json::json!({"op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4}), + serde_json::json!({"op":"reduce","symbol":"TEST","order_id":1,"quantity":0.4}), + serde_json::json!({"op":"replace","symbol":"TEST","order_id":1,"quantity":3.5}), + serde_json::json!({"op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":1.4}), + ].into_iter().enumerate() { + let event = serde_json::from_value::(value).unwrap(); + let observation = adapter.apply(&event, index as u64 + 1).unwrap(); + assert_eq!(observation.outcome.status, "applied"); + } + let state = adapter.snapshot().unwrap(); + assert_eq!(state.books[0].bids.len(), 1); + assert_eq!(state.books[0].bids[0].remaining_quantity, "2"); + } + + #[test] + fn nonrepresentable_quantities_preserve_existing_state() { + let mut adapter = Adapter::new(ConfigWire::default()).unwrap(); + let first = serde_json::from_value::(serde_json::json!({ + "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":2.4,"owner":9 + })).unwrap(); + let initial = adapter.apply(&first, 1).unwrap(); + for quantity in ["0.0000000000001", "18446745"] { + for op in ["new", "reduce", "replace"] { + let event = serde_json::from_value::(serde_json::json!({ + "op":op,"symbol":"TEST","order_id":if op == "new" {2} else {1}, + "side":"BUY","price":100,"quantity":serde_json::Number::from_str(quantity).unwrap() + })).unwrap(); + let observation = adapter.apply(&event, 2).unwrap(); + assert_eq!(observation.outcome.status, "rejected"); + assert_eq!( + observation.outcome.reason, + Some(if op == "replace" { + "INVALID_REPLACEMENT" + } else { + "INVALID_ORDER" + }) + ); + assert_eq!(observation.state_hash, initial.state_hash); + assert!(observation.trades.is_empty()); + } + } } #[test] diff --git a/integrations/rust_protocol/README.md b/integrations/rust_protocol/README.md index 215605c..6e1a4ee 100644 --- a/integrations/rust_protocol/README.md +++ b/integrations/rust_protocol/README.md @@ -4,6 +4,26 @@ This private crate owns Tracebook protocol v1 framing, validation, canonical book-state serialization, hashing, and the stdin/stdout server loop used by the native Rust engine adapters. +The crate also supplies the exact `QuantityEncoding` boundary shared by +`orderbook_rs` and `gocronx_matcher`. Quantities become integer native lots at a +fixed twelve decimal places, independently of `quantity_decimal_places`. +That protocol setting controls half-even rounding of output quantities only; +it never changes submitted order, reduction, or replacement quantities. + +The individual input range is positive `u64` lots: from `0.000000000001` through +`18446744.073709551615`. Inputs with additional non-zero decimal places or a +larger scaled value are rejected before native submission. Scientific notation +and insignificant trailing zeros are handled exactly. Parsing uses checked +integer arithmetic rather than a decimal parser that could silently round a +long mantissa into range. Each native engine retains its own capacity limits. + +The fixed twelve-place scale preserves the native lots used by existing +default-precision qualification workloads. Other numeric configurations remain +unqualified. For example, output precision eighteen exposes the native exact +remainder `0.2` after `0.3 - 0.1`, while the binary64 reference reports +`0.19999999999999998`. The adapters preserve native results, including such +differences; they do not imitate the reference's arithmetic to obtain a pass. + Engine-specific crates implement `EngineAdapter` and provide a constructor. They must not copy protocol frames or server behavior locally. This keeps every Rust candidate on the same observation contract while leaving matching and diff --git a/integrations/rust_protocol/src/lib.rs b/integrations/rust_protocol/src/lib.rs index 337572e..c3ee386 100644 --- a/integrations/rust_protocol/src/lib.rs +++ b/integrations/rust_protocol/src/lib.rs @@ -3,8 +3,10 @@ use serde_json::{Number, Value}; use sha2::{Digest, Sha256}; use std::io::{self, Write}; +mod quantity; mod server; +pub use quantity::QuantityEncoding; pub use server::{EngineAdapter, EngineIdentity, run}; pub const PROTOCOL_NAME: &str = "tracebook.conformance"; diff --git a/integrations/rust_protocol/src/quantity.rs b/integrations/rust_protocol/src/quantity.rs new file mode 100644 index 0000000..ead2ae1 --- /dev/null +++ b/integrations/rust_protocol/src/quantity.rs @@ -0,0 +1,179 @@ +use serde_json::Number; + +const NATIVE_DECIMAL_PLACES: u32 = 12; + +/// Exact u64 lots shared by the two native Rust adapters. +/// Output precision never changes the quantity submitted to either engine. +#[derive(Clone)] +pub struct QuantityEncoding { + output_decimal_places: u32, +} + +impl QuantityEncoding { + pub fn new(output_decimal_places: u32) -> Result { + if output_decimal_places > 18 { + return Err("quantity_decimal_places must be between 0 and 18".to_string()); + } + Ok(Self { + output_decimal_places, + }) + } + + pub fn units(&self, value: &Number) -> Result { + let text = value.to_string(); + if text.starts_with('-') { + return Err("quantity must be positive".to_string()); + } + let range_error = || "quantity exceeds the adapter's integer range".to_string(); + let (mantissa, exponent) = match text.split_once(['e', 'E']) { + Some((mantissa, exponent)) => ( + mantissa, + exponent.parse::().map_err(|_| range_error())?, + ), + None => (text.as_str(), 0), + }; + let fractional_places = mantissa.split_once('.').map_or(0, |(_, part)| part.len()); + let digits = mantissa.replace('.', ""); + let significant = digits.trim_start_matches('0'); + if significant.is_empty() { + return Err("quantity must be positive".to_string()); + } + let coefficient = significant.trim_end_matches('0'); + let trailing_zeros = significant.len() - coefficient.len(); + let shift = exponent + .checked_sub(i64::try_from(fractional_places).map_err(|_| range_error())?) + .and_then(|value| value.checked_add(i64::from(NATIVE_DECIMAL_PLACES))) + .and_then(|value| value.checked_add(i64::try_from(trailing_zeros).ok()?)) + .ok_or_else(range_error)?; + if shift < 0 { + return Err(format!( + "quantity requires more than {NATIVE_DECIMAL_PLACES} native decimal places" + )); + } + // Parse only the significant digits after removing exact decimal zeros. + // No binary float or fixed-precision decimal parser may round this input. + let coefficient = coefficient.parse::().map_err(|_| range_error())?; + let multiplier = u32::try_from(shift) + .ok() + .and_then(|shift| 10_u64.checked_pow(shift)) + .ok_or_else(range_error)?; + coefficient.checked_mul(multiplier).ok_or_else(range_error) + } + + pub fn format(&self, units: u64) -> String { + let places = self.output_decimal_places.min(NATIVE_DECIMAL_PLACES); + let divisor = 10_u64.pow(NATIVE_DECIMAL_PLACES - places); + let mut rounded = units / divisor; + let remainder = units % divisor; + if divisor > 1 + && (remainder > divisor / 2 || (remainder == divisor / 2 && rounded % 2 == 1)) + { + // With divisor >= 10, incrementing the quotient cannot overflow. + rounded += 1; + } + let scale = 10_u64.pow(places); + let whole = rounded / scale; + let fraction = rounded % scale; + if fraction == 0 { + return whole.to_string(); + } + format!("{whole}.{fraction:0width$}", width = places as usize) + .trim_end_matches('0') + .to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + fn number(text: &str) -> Number { + Number::from_str(text).unwrap() + } + + #[test] + fn output_precision_does_not_round_submitted_lots() { + for places in [0, 3, 12, 18] { + let encoding = QuantityEncoding::new(places).unwrap(); + assert_eq!( + encoding.units(&number("1.2345")).unwrap(), + 1_234_500_000_000 + ); + assert_eq!(encoding.units(&number("1e-12")).unwrap(), 1); + } + } + + #[test] + fn scientific_notation_and_insignificant_zeros_remain_exact() { + let encoding = QuantityEncoding::new(12).unwrap(); + for (text, units) in [ + ("1E+1", 10_000_000_000_000), + ("1.234e-9", 1_234), + ( + "1234000000000000000000000000000000000e-36", + 1_234_000_000_000, + ), + ("1.000000000000000000000000000000000000", 1_000_000_000_000), + ] { + assert_eq!(encoding.units(&number(text)).unwrap(), units, "{text}"); + } + } + + #[test] + fn nonrepresentable_inputs_cannot_round_into_the_native_domain() { + let encoding = QuantityEncoding::new(0).unwrap(); + for text in [ + "0.0000000000011", + "1e-13", + "1.00000000000000000000000000001", + "100000000000000000000000000001e-29", + ] { + assert!( + encoding + .units(&number(text)) + .unwrap_err() + .contains("more than 12"), + "{text}" + ); + } + } + + #[test] + fn positive_u64_range_is_checked_without_truncation() { + let encoding = QuantityEncoding::new(12).unwrap(); + assert_eq!( + encoding.units(&number("18446744.073709551615")).unwrap(), + u64::MAX + ); + for text in [ + "0", + "-1", + "18446744.073709551616", + "1e99", + "1e9999999999999999999", + ] { + assert!(encoding.units(&number(text)).is_err(), "{text}"); + } + assert_eq!(encoding.format(u64::MAX), "18446744.073709551615"); + } + + #[test] + fn observation_rounding_is_half_even_and_canonical() { + for (places, text, expected) in [ + (3, "1.2345", "1.234"), + (3, "1.2355", "1.236"), + (0, "2.5", "2"), + (0, "3.5", "4"), + (18, "0.2", "0.2"), + (12, "1e-12", "0.000000000001"), + ] { + let encoding = QuantityEncoding::new(places).unwrap(); + assert_eq!( + encoding.format(encoding.units(&number(text)).unwrap()), + expected + ); + } + assert!(QuantityEncoding::new(19).is_err()); + } +} diff --git a/src/tracebook/book_replay/artifacts.py b/src/tracebook/book_replay/artifacts.py new file mode 100644 index 0000000..f762a14 --- /dev/null +++ b/src/tracebook/book_replay/artifacts.py @@ -0,0 +1,204 @@ +"""Reserve and atomically publish new L3 report and trace files.""" + +from __future__ import annotations + +import os +import uuid +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Iterable, Iterator, Optional + +from .model import BookReplayError + + +def _identity(stat: os.stat_result) -> tuple[int, int]: + return stat.st_dev, stat.st_ino + + +def _write_payload(descriptor: int, payload: bytes) -> None: + with os.fdopen(os.dup(descriptor), "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + + +class OutputReservation: + """Own a sidecar reservation while the destination itself stays absent.""" + + def __init__(self, destination: str | Path) -> None: + path = Path(destination).expanduser() + self._requested_target = path.absolute() + self.target = path.parent.resolve() / path.name + self._lock_name = f".{self.target.name}.tracebook-in-progress" + self._stage_name = f".tracebook-stage-{uuid.uuid4().hex}" + self._parent_fd: Optional[int] = None + self._lock_fd: Optional[int] = None + self._stage_fd: Optional[int] = None + self._published = False + + def __enter__(self) -> "OutputReservation": + if not all(operation in os.supports_dir_fd for operation in (os.open, os.stat, os.link)): + raise BookReplayError( + "L3 artifact publication requires descriptor-relative file operations" + ) + try: + self.target.parent.mkdir(parents=True, exist_ok=True) + self._parent_fd = os.open( + self.target.parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + self._require_absent() + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + self._lock_fd = os.open(self._lock_name, flags, 0o600, dir_fd=self._parent_fd) + self._stage_fd = os.open(self._stage_name, flags, 0o600, dir_fd=self._parent_fd) + self.validate() + except BaseException: + try: + self.close() + except OSError: + pass + raise + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + try: + self.close() + except OSError: + if exc_type is None: + raise + + def _require_absent(self) -> None: + try: + os.stat(self.target.name, dir_fd=self._parent_fd, follow_symlinks=False) + except FileNotFoundError: + return + raise BookReplayError(f"book-replay output already exists: {self.target}") + + def _owns_entry(self, name: str, descriptor: Optional[int]) -> bool: + if self._parent_fd is None or descriptor is None: + return False + try: + return _identity( + os.stat(name, dir_fd=self._parent_fd, follow_symlinks=False) + ) == _identity(os.fstat(descriptor)) + except OSError: + return False + + def validate(self) -> None: + if self._parent_fd is None: + raise BookReplayError("book-replay output reservation is closed") + if ( + not self._same_parent() + or not self._owns_entry(self._lock_name, self._lock_fd) + or not self._owns_entry(self._stage_name, self._stage_fd) + ): + raise BookReplayError(f"book-replay output reservation changed: {self.target}") + self._require_absent() + + def _same_parent(self) -> bool: + if self._parent_fd is None: + return False + try: + expected = _identity(os.fstat(self._parent_fd)) + return ( + _identity(self.target.parent.stat(follow_symlinks=False)) == expected + and _identity(self._requested_target.parent.stat()) == expected + ) + except OSError: + return False + + def stage(self, payload: bytes) -> None: + self.validate() + if self._stage_fd is None: + raise BookReplayError("book-replay output reservation is closed") + _write_payload(self._stage_fd, payload) + + def publish(self) -> None: + self.validate() + # A hard link publishes the complete staged inode atomically and fails + # if anything appeared at the destination after validation. + # Mark the attempt first: interruption immediately after a successful + # link still needs rollback, which checks the destination's inode. + self._published = True + os.link( + self._stage_name, + self.target.name, + src_dir_fd=self._parent_fd, + dst_dir_fd=self._parent_fd, + follow_symlinks=False, + ) + self.validate_published() + + def validate_published(self) -> None: + if ( + not self._published + or not self._same_parent() + or not self._owns_entry(self.target.name, self._stage_fd) + ): + raise BookReplayError(f"book-replay output changed during publication: {self.target}") + + def rollback(self) -> None: + if self._published and self._owns_entry(self.target.name, self._stage_fd): + os.unlink(self.target.name, dir_fd=self._parent_fd) + self._published = False + + def close(self) -> None: + error: Optional[OSError] = None + for name, descriptor in ( + (self._stage_name, self._stage_fd), + (self._lock_name, self._lock_fd), + ): + if descriptor is not None: + try: + if self._owns_entry(name, descriptor): + os.unlink(name, dir_fd=self._parent_fd) + except OSError as exc: + error = error or exc + finally: + try: + os.close(descriptor) + except OSError as exc: + error = error or exc + self._stage_fd = self._lock_fd = None + if self._parent_fd is not None: + try: + os.close(self._parent_fd) + except OSError as exc: + error = error or exc + self._parent_fd = None + if error is not None: + raise error + + +@contextmanager +def reserve_outputs(*destinations: Optional[str]) -> Iterator[dict[str, OutputReservation]]: + """Reserve every output before candidate work, including conditional traces.""" + with ExitStack() as stack: + outputs = { + destination: stack.enter_context(OutputReservation(destination)) + for destination in destinations + if destination is not None + } + # Also catch a destination colliding with another output's sidecar. + for output in outputs.values(): + output.validate() + yield outputs + + +def publish_outputs(payloads: Iterable[tuple[OutputReservation, bytes]]) -> None: + """Stage all bytes, then publish in order; roll back this attempt on failure.""" + prepared = list(payloads) + try: + for output, payload in prepared: + output.stage(payload) + for output, _payload in prepared: + output.publish() + for output, _payload in prepared: + output.validate_published() + except BaseException: + for output, _payload in reversed(prepared): + try: + output.rollback() + except OSError: + pass + raise diff --git a/src/tracebook/book_replay/cli.py b/src/tracebook/book_replay/cli.py index ce89978..2b4ab45 100644 --- a/src/tracebook/book_replay/cli.py +++ b/src/tracebook/book_replay/cli.py @@ -11,6 +11,7 @@ from typing import List, Optional from .._version import __version__ +from .artifacts import OutputReservation, publish_outputs, reserve_outputs from .campaign import run_book_replay_campaign from .compare import run_book_replay from .external import ExternalBookReplayAdapterFactory @@ -90,45 +91,49 @@ def _copy_sample(destination: str) -> Path: target = root / f"{PROFILE_NAME}.jsonl" if root.exists() and not root.is_dir(): raise BookReplayError("sample destination must be a directory") - if target.exists(): - raise BookReplayError(f"sample trace already exists: {target}") - root.mkdir(parents=True, exist_ok=True) source = resources.files("tracebook.book_replay.fixtures").joinpath(f"{PROFILE_NAME}.jsonl") - target.write_bytes(source.read_bytes()) + with reserve_outputs(str(target)) as outputs: + publish_outputs([(outputs[str(target)], source.read_bytes())]) return target -def _emit(payload: dict, output: Optional[str]) -> None: +def _emit( + payload: dict, + output: Optional[str], + outputs: dict[str, OutputReservation], + *, + events=None, + events_output: Optional[str] = None, +) -> None: rendered = json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n" + files = [] + if events_output is not None and events is not None: + event_bytes = "".join( + json.dumps(event.to_dict(), sort_keys=True, separators=(",", ":"), allow_nan=False) + + "\n" + for event in events + ).encode("utf-8") + files.append((outputs[events_output], event_bytes)) + if output is not None: + files.append((outputs[output], rendered.encode("utf-8"))) + # Publish the report last so a successful report never precedes its trace. + publish_outputs(files) if output is None: print(rendered, end="") - return - path = Path(output) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(rendered, encoding="utf-8") - print(f"Report written: {path}") - - -def _write_events(events, output: str) -> None: - path = Path(output) - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as handle: - for event in events: - handle.write( - json.dumps( - event.to_dict(), - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - + "\n" - ) + else: + print(f"Report written: {Path(output)}") def _require_distinct_paths(*paths: Optional[str]) -> None: resolved = [Path(path).expanduser().resolve() for path in paths if path is not None] - if len(resolved) != len(set(resolved)): - raise BookReplayError("input and output paths must be distinct") + if len(resolved) != len(set(resolved)) or any( + left in right.parents or right in left.parents + for index, left in enumerate(resolved) + for right in resolved[index + 1 :] + ): + raise BookReplayError( + "input and output paths must be distinct and cannot contain one another" + ) def main(argv: Optional[List[str]] = None) -> int: @@ -142,17 +147,23 @@ def main(argv: Optional[List[str]] = None) -> int: if args.command == "campaign": _require_distinct_paths(args.output, args.reduced_events_output) - campaign = run_book_replay_campaign( - _candidate_factory(args), - seed=args.seed, - traces=args.traces, - events_per_trace=args.events_per_trace, - max_minimize_runs=args.max_minimize_runs, - ) - _emit(campaign.to_dict(), args.output) - if campaign.failure is not None and args.reduced_events_output is not None: - _write_events(campaign.failure.minimization.events, args.reduced_events_output) - print(f"Reduced events written: {Path(args.reduced_events_output)}") + with reserve_outputs(args.output, args.reduced_events_output) as outputs: + campaign = run_book_replay_campaign( + _candidate_factory(args), + seed=args.seed, + traces=args.traces, + events_per_trace=args.events_per_trace, + max_minimize_runs=args.max_minimize_runs, + ) + _emit( + campaign.to_dict(), + args.output, + outputs, + events=campaign.failure.minimization.events if campaign.failure else None, + events_output=args.reduced_events_output, + ) + if campaign.failure is not None and args.reduced_events_output is not None: + print(f"Reduced events written: {Path(args.reduced_events_output)}") if campaign.failure is None: return 0 return 2 if campaign.failure.minimization.report.operational_failure else 1 @@ -160,26 +171,31 @@ def main(argv: Optional[List[str]] = None) -> int: events_path = Path(args.events).expanduser().resolve() events_output = args.events_output if args.command == "minimize" else None _require_distinct_paths(str(events_path), args.output, events_output) - if args.command == "minimize": - minimization = minimize_book_replay_failure( + with reserve_outputs(args.output, events_output) as outputs: + if args.command == "minimize": + minimization = minimize_book_replay_failure( + load_book_replay_events(str(events_path)), + _candidate_factory(args), + config=BookReplayConfig(args.profile), + max_runs=args.max_runs, + trace_name=str(events_path), + ) + _emit( + minimization.to_dict(), + args.output, + outputs, + events=minimization.events, + events_output=args.events_output, + ) + print(f"Minimized events written: {Path(args.events_output)}") + return 2 if minimization.report.operational_failure else 1 + report = run_book_replay( load_book_replay_events(str(events_path)), _candidate_factory(args), config=BookReplayConfig(args.profile), - max_runs=args.max_runs, trace_name=str(events_path), ) - _write_events(minimization.events, args.events_output) - _emit(minimization.to_dict(), args.output) - print(f"Minimized events written: {Path(args.events_output)}") - return 2 if minimization.report.operational_failure else 1 - report = run_book_replay( - load_book_replay_events(str(events_path)), - _candidate_factory(args), - config=BookReplayConfig(args.profile), - trace_name=str(events_path), - ) - payload = report.to_dict() - _emit(payload, args.output) + _emit(report.to_dict(), args.output, outputs) if report.operational_failure: return 2 return 0 if report.conformant else 1 diff --git a/tests/test_book_replay_artifacts.py b/tests/test_book_replay_artifacts.py new file mode 100644 index 0000000..c229d66 --- /dev/null +++ b/tests/test_book_replay_artifacts.py @@ -0,0 +1,406 @@ +"""L3 output failures must not truncate prior evidence or run invalid targets.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tracebook.book_replay import BookReplayEvent, ReferenceBookReplayAdapter +from tracebook.book_replay import artifacts, cli + +EVENT = BookReplayEvent("add", "TEST", 1, "BUY", "100", "2") +PAYLOAD = {"artifact_type": "tracebook.book-replay.report", "conformant": True} + + +def _trace(path: Path) -> None: + path.write_text(json.dumps(EVENT.to_dict()) + "\n", encoding="utf-8") + + +def _command(command: str, trace: Path, report: Path, reduced: Path) -> list[str]: + arguments = [command] + if command != "campaign": + arguments.append(str(trace)) + arguments.extend(("--output", str(report))) + if command == "campaign": + arguments.extend(("--reduced-events-output", str(reduced))) + elif command == "minimize": + arguments.extend(("--events-output", str(reduced))) + arguments.extend(("--candidate-cmd", "unused-adapter")) + return arguments + + +def _forbid_candidate(monkeypatch) -> None: + def fail(_args): + pytest.fail("candidate factory reached despite invalid output paths") + + monkeypatch.setattr(cli, "_candidate_factory", fail) + + +def _stub_result(monkeypatch, command: str) -> None: + monkeypatch.setattr(cli, "_candidate_factory", lambda _args: object()) + report = SimpleNamespace(conformant=True, operational_failure=False, to_dict=lambda: PAYLOAD) + minimization = SimpleNamespace(events=[EVENT], report=report, to_dict=lambda: PAYLOAD) + result = { + "run": report, + "minimize": minimization, + "campaign": SimpleNamespace( + failure=SimpleNamespace(minimization=minimization), to_dict=lambda: PAYLOAD + ), + }[command] + function = { + "run": "run_book_replay", + "minimize": "minimize_book_replay_failure", + "campaign": "run_book_replay_campaign", + }[command] + monkeypatch.setattr(cli, function, lambda *_args, **_kwargs: result) + + +@pytest.mark.parametrize("command", ("run", "minimize", "campaign")) +@pytest.mark.parametrize("existing_kind", ("file", "directory", "symlink", "dangling-symlink")) +def test_cli_refuses_existing_report_before_candidate( + tmp_path, monkeypatch, command, existing_kind +): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + if existing_kind == "file": + report.write_bytes(b"prior evidence\n") + elif existing_kind == "directory": + report.mkdir() + else: + report.symlink_to(tmp_path / "prior.json") + if existing_kind == "symlink": + (tmp_path / "prior.json").write_bytes(b"prior evidence\n") + before = set(tmp_path.iterdir()) + _forbid_candidate(monkeypatch) + + assert cli.main(_command(command, trace, report, reduced)) == 2 + + assert set(tmp_path.iterdir()) == before + if existing_kind == "file": + assert report.read_bytes() == b"prior evidence\n" + if existing_kind == "symlink": + assert report.is_symlink() + assert report.read_bytes() == b"prior evidence\n" + if existing_kind == "dangling-symlink": + assert report.is_symlink() + assert not (tmp_path / "prior.json").exists() + + +@pytest.mark.parametrize("command", ("minimize", "campaign")) +def test_cli_reserves_conditional_reduced_output_before_candidate(tmp_path, monkeypatch, command): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + reduced.write_bytes(b"prior reduced trace\n") + _forbid_candidate(monkeypatch) + + assert cli.main(_command(command, trace, report, reduced)) == 2 + + assert reduced.read_bytes() == b"prior reduced trace\n" + assert set(tmp_path.iterdir()) == {trace, reduced} + + +@pytest.mark.parametrize("command", ("minimize", "campaign")) +@pytest.mark.parametrize( + "collision", ("same", "nested", "reverse-nested", "sidecar", "reverse-sidecar") +) +def test_cli_rejects_colliding_output_paths_before_candidate( + tmp_path, monkeypatch, command, collision +): + trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json" + _trace(trace) + reduced = { + "same": tmp_path / "unused" / ".." / "report.json", + "nested": report / "reduced.jsonl", + "reverse-nested": tmp_path, + "sidecar": tmp_path / ".report.json.tracebook-in-progress", + "reverse-sidecar": report, + }[collision] + if collision == "reverse-sidecar": + report = tmp_path / ".report.json.tracebook-in-progress" + _forbid_candidate(monkeypatch) + + assert cli.main(_command(command, trace, report, reduced)) == 2 + + assert set(tmp_path.iterdir()) == {trace} + + +@pytest.mark.parametrize("command", ("run", "minimize", "campaign")) +def test_cli_rejects_non_directory_parent_before_candidate(tmp_path, monkeypatch, command): + trace, parent = tmp_path / "trace.jsonl", tmp_path / "not-a-directory" + _trace(trace) + parent.write_bytes(b"keep this file\n") + _forbid_candidate(monkeypatch) + + assert cli.main(_command(command, trace, parent / "report.json", tmp_path / "reduced")) == 2 + assert parent.read_bytes() == b"keep this file\n" + + +def test_cli_refuses_another_active_reservation_before_candidate(tmp_path, monkeypatch): + trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json" + _trace(trace) + _forbid_candidate(monkeypatch) + with artifacts.reserve_outputs(str(report)): + before = set(tmp_path.iterdir()) + assert cli.main(_command("run", trace, report, tmp_path / "unused")) == 2 + assert set(tmp_path.iterdir()) == before + assert set(tmp_path.iterdir()) == {trace} + + +@pytest.mark.parametrize("command", ("run", "minimize", "campaign", "sample")) +def test_partial_stage_write_never_publishes_an_artifact(tmp_path, monkeypatch, command): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + if command != "sample": + _stub_result(monkeypatch, command) + + def partial_write(descriptor, payload): + os.write(descriptor, payload[:5]) + raise OSError("injected partial write failure") + + monkeypatch.setattr(artifacts, "_write_payload", partial_write) + arguments = ( + ["sample", str(tmp_path)] + if command == "sample" + else _command(command, trace, report, reduced) + ) + + assert cli.main(arguments) == 2 + assert set(tmp_path.iterdir()) == {trace} + + +@pytest.mark.parametrize("command", ("minimize", "campaign")) +def test_second_publication_failure_rolls_back_first_file(tmp_path, monkeypatch, command): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + _stub_result(monkeypatch, command) + link = os.link + published = [] + + def fail_second(source, target, **kwargs): + published.append(target) + if len(published) == 2: + raise OSError("injected report publication failure") + link(source, target, **kwargs) + + monkeypatch.setattr(artifacts.os, "link", fail_second) + # The runtime supports descriptor-relative links; the wrapper remains the + # same operation for this injected failure. + monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {fail_second}) + + assert cli.main(_command(command, trace, report, reduced)) == 2 + assert published == [reduced.name, report.name] + assert set(tmp_path.iterdir()) == {trace} + + +def test_late_destination_creation_is_never_overwritten(tmp_path, monkeypatch): + trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json" + _trace(trace) + _stub_result(monkeypatch, "run") + link = os.link + + def race_link(source, target, **kwargs): + report.write_bytes(b"another writer's evidence\n") + link(source, target, **kwargs) + + monkeypatch.setattr(artifacts.os, "link", race_link) + monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {race_link}) + + assert cli.main(_command("run", trace, report, tmp_path / "unused")) == 2 + assert report.read_bytes() == b"another writer's evidence\n" + assert set(tmp_path.iterdir()) == {trace, report} + + +def test_replaced_output_parent_cannot_redirect_publication(tmp_path, monkeypatch): + trace = tmp_path / "trace.jsonl" + output_parent, moved_parent, other_parent = ( + tmp_path / name for name in ("out", "moved", "other") + ) + _trace(trace) + other_parent.mkdir() + _stub_result(monkeypatch, "run") + + def replace_parent(*_args, **_kwargs): + output_parent.rename(moved_parent) + output_parent.symlink_to(other_parent, target_is_directory=True) + return SimpleNamespace(to_dict=lambda: PAYLOAD, conformant=True, operational_failure=False) + + monkeypatch.setattr(cli, "run_book_replay", replace_parent) + + assert cli.main(_command("run", trace, output_parent / "report.json", tmp_path / "unused")) == 2 + assert list(other_parent.iterdir()) == [] + assert list(moved_parent.iterdir()) == [] + assert output_parent.is_symlink() + + +def test_retargeted_parent_alias_cannot_publish_at_an_invisible_old_path(tmp_path, monkeypatch): + trace = tmp_path / "trace.jsonl" + original, other, alias = (tmp_path / name for name in ("original", "other", "alias")) + _trace(trace) + original.mkdir() + other.mkdir() + alias.symlink_to(original, target_is_directory=True) + _stub_result(monkeypatch, "run") + + def retarget_alias(*_args, **_kwargs): + alias.unlink() + alias.symlink_to(other, target_is_directory=True) + return SimpleNamespace(to_dict=lambda: PAYLOAD, conformant=True, operational_failure=False) + + monkeypatch.setattr(cli, "run_book_replay", retarget_alias) + + assert cli.main(_command("run", trace, alias / "report.json", tmp_path / "unused")) == 2 + assert list(original.iterdir()) == [] + assert list(other.iterdir()) == [] + assert alias.resolve() == other + + +def test_replaced_reduced_file_invalidates_and_rolls_back_the_report(tmp_path, monkeypatch): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + _stub_result(monkeypatch, "minimize") + link = os.link + + def replace_reduced_before_report(source, target, **kwargs): + if target == report.name: + reduced.unlink() + reduced.write_bytes(b"another writer's replacement\n") + link(source, target, **kwargs) + + monkeypatch.setattr(artifacts.os, "link", replace_reduced_before_report) + monkeypatch.setattr( + artifacts.os, "supports_dir_fd", os.supports_dir_fd | {replace_reduced_before_report} + ) + + assert cli.main(_command("minimize", trace, report, reduced)) == 2 + assert not report.exists() + assert reduced.read_bytes() == b"another writer's replacement\n" + assert set(tmp_path.iterdir()) == {trace, reduced} + + +def test_interruption_after_report_link_rolls_back_both_published_files(tmp_path, monkeypatch): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + _stub_result(monkeypatch, "minimize") + link = os.link + + def interrupt_after_link(source, target, **kwargs): + link(source, target, **kwargs) + if target == report.name: + raise KeyboardInterrupt + + monkeypatch.setattr(artifacts.os, "link", interrupt_after_link) + monkeypatch.setattr( + artifacts.os, "supports_dir_fd", os.supports_dir_fd | {interrupt_after_link} + ) + + with pytest.raises(KeyboardInterrupt): + cli.main(_command("minimize", trace, report, reduced)) + assert set(tmp_path.iterdir()) == {trace} + + +def test_cleanup_failure_still_closes_all_descriptors_and_other_sidecars(tmp_path, monkeypatch): + reservation = artifacts.OutputReservation(tmp_path / "report.json") + reservation.__enter__() + descriptors = (reservation._stage_fd, reservation._lock_fd, reservation._parent_fd) + unlink = os.unlink + + def fail_stage_cleanup(name, **kwargs): + if name == reservation._stage_name: + raise OSError("injected temporary file cleanup failure") + unlink(name, **kwargs) + + monkeypatch.setattr(artifacts.os, "unlink", fail_stage_cleanup) + + with pytest.raises(OSError, match="cleanup failure"): + reservation.close() + for descriptor in descriptors: + with pytest.raises(OSError): + os.fstat(descriptor) + assert {path.name for path in tmp_path.iterdir()} == {reservation._stage_name} + + +def test_trace_serialization_failure_precedes_any_artifact_write(tmp_path, monkeypatch): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + _stub_result(monkeypatch, "minimize") + + def bad_event(): + raise ValueError("injected event serialization error") + + result = SimpleNamespace( + to_dict=lambda: PAYLOAD, + events=[EVENT, SimpleNamespace(to_dict=bad_event)], + ) + monkeypatch.setattr(cli, "minimize_book_replay_failure", lambda *_args, **_kwargs: result) + monkeypatch.setattr( + artifacts, + "_write_payload", + lambda *_args: pytest.fail("writing before serialization finished"), + ) + + assert cli.main(_command("minimize", trace, report, reduced)) == 2 + assert set(tmp_path.iterdir()) == {trace} + + +@pytest.mark.parametrize("command", ("run", "minimize", "campaign")) +def test_successful_publication_preserves_existing_json_bytes(tmp_path, monkeypatch, command): + trace, report, reduced = ( + tmp_path / name for name in ("trace.jsonl", "report.json", "reduced.jsonl") + ) + _trace(trace) + _stub_result(monkeypatch, command) + + assert cli.main(_command(command, trace, report, reduced)) == (0 if command == "run" else 1) + + assert report.read_bytes() == (json.dumps(PAYLOAD, sort_keys=True, indent=2) + "\n").encode() + if command != "run": + assert ( + reduced.read_bytes() + == (json.dumps(EVENT.to_dict(), sort_keys=True, separators=(",", ":")) + "\n").encode() + ) + assert not list(tmp_path.glob(".*")) + + +def test_passing_campaign_removes_unused_reduced_reservation(tmp_path, monkeypatch): + report, reduced = tmp_path / "campaign.json", tmp_path / "reduced.jsonl" + monkeypatch.setattr(cli, "_candidate_factory", lambda _args: ReferenceBookReplayAdapter) + + assert ( + cli.main( + [ + "campaign", + "--output", + str(report), + "--reduced-events-output", + str(reduced), + "--traces", + "1", + "--events-per-trace", + "18", + "--candidate-cmd", + "unused", + ] + ) + == 0 + ) + + assert json.loads(report.read_text())["conformant"] is True + assert set(tmp_path.iterdir()) == {report} diff --git a/tests/test_conformance.py b/tests/test_conformance.py index ce8fa42..276f902 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -595,7 +595,7 @@ def test_external_adapter_rejects_an_invalid_complete_handshake(): report = run_conformance( [], lambda config: ExternalProcessAdapter( - [sys.executable, "-c", script], config, timeout_seconds=1 + [sys.executable, "-c", script], config, timeout_seconds=10 ), ) diff --git a/tests/test_integration_workflow_coverage.py b/tests/test_integration_workflow_coverage.py new file mode 100644 index 0000000..f6d2971 --- /dev/null +++ b/tests/test_integration_workflow_coverage.py @@ -0,0 +1,134 @@ +"""Keep native CI attached to the reference/runtime code it actually executes.""" + +import ast +import os +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = { + "intrepid-orderbook": "intrepid_orderbook", + "nautilus-trader-book-replay": "nautilus_trader", + "orderbook-rs": "orderbook_rs", + "gocronx-matcher": "gocronx_matcher", + "python-matching-engine": "python_matching_engine", +} + + +def _pull_request_paths(workflow): + source = (ROOT / ".github" / "workflows" / f"{workflow}.yml").read_text(encoding="utf-8") + # Read this deliberately simple YAML sequence without adding PyYAML to the + # dependency-free package's test requirements. Do not search job bodies, + # where a mentioned path cannot cause the workflow to run. + match = re.search(r"^ pull_request:\n paths:\n((?: - .+\n)+)", source, re.M) + assert match is not None, f"{workflow} must declare pull_request.paths" + patterns = [ast.literal_eval(line.removeprefix(" - ")) for line in match[1].splitlines()] + assert all(isinstance(pattern, str) and not pattern.startswith("!") for pattern in patterns) + return patterns + + +def _matches(path, pattern): + # The checked workflows use only literal paths and positive * / ** globs. + # Unlike fnmatch, a single * must not match a directory separator. + expression = re.escape(pattern).replace(r"\*\*", ".*").replace(r"\*", "[^/]*") + return re.fullmatch(expression, path) is not None + + +def _require_covered(workflow, paths): + patterns = _pull_request_paths(workflow) + missing = sorted(path for path in paths if not any(_matches(path, p) for p in patterns)) + assert not missing, f"{workflow} misses direct runtime inputs: {missing}" + + +def _python_sources(relative_root): + return [path.relative_to(ROOT).as_posix() for path in (ROOT / relative_root).rglob("*.py")] + + +@pytest.mark.parametrize("workflow", WORKFLOWS) +def test_integration_workflows_cover_installed_reference_runtime(workflow): + # Even L3 imports conformance.model and tracebook's package initializer. + # Every CLOB proof directly runs the shared Python reference matching engine. + paths = [ + "pyproject.toml", + "MANIFEST.in", + "src/tracebook/__init__.py", + "src/tracebook/_version.py", + f".github/workflows/{workflow}.yml", + "tests/test_integration_workflow_coverage.py", + ] + for package in ("core", "events", "conformance"): + paths.extend(_python_sources(f"src/tracebook/{package}")) + paths.extend( + path.relative_to(ROOT).as_posix() + for path in (ROOT / "src/tracebook/conformance/fixtures").rglob("*") + if path.suffix in {".json", ".jsonl"} + ) + _require_covered(workflow, paths) + + +@pytest.mark.parametrize("workflow", WORKFLOWS) +def test_integration_workflows_cover_own_source_pins_and_retained_evidence(workflow): + integration = ROOT / "integrations" / WORKFLOWS[workflow] + paths = [] + for directory, directories, filenames in os.walk(integration): + directories[:] = [name for name in directories if name not in {"target", "__pycache__"}] + for name in filenames: + path = Path(directory) / name + if path.suffix in {".go", ".rs", ".py", ".json", ".jsonl"} or name in { + "go.mod", + "go.sum", + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + }: + paths.append(path.relative_to(ROOT).as_posix()) + assert paths + _require_covered(workflow, paths) + + +@pytest.mark.parametrize("workflow", ["orderbook-rs", "gocronx-matcher"]) +def test_rust_workflows_cover_the_shared_protocol_crate(workflow): + paths = ["integrations/rust_protocol/Cargo.toml", "integrations/rust_protocol/Cargo.lock"] + paths.extend( + path.relative_to(ROOT).as_posix() + for path in (ROOT / "integrations/rust_protocol/src").rglob("*.rs") + ) + _require_covered(workflow, paths) + + +def test_nautilus_workflow_covers_book_replay_and_executed_test_helpers(): + paths = _python_sources("src/tracebook/book_replay") + paths.extend( + [ + "src/tracebook/book_replay/fixtures/l3-book-replay-v1.jsonl", + "examples/book_replay_adapter.py", + "integrations/__init__.py", + "integrations/python_matching_engine/__init__.py", + "integrations/python_matching_engine/adapter.py", + "tests/conftest.py", + "tests/test_book_replay.py", + "tests/test_book_replay_campaign.py", + "tests/test_book_replay_artifacts.py", + "tests/fixtures/faulty_book_replay_adapter.py", + "tests/test_external_integrations.py", + ] + ) + _require_covered("nautilus-trader-book-replay", paths) + + +def test_orderbook_rs_workflow_covers_the_executed_flash_bridge(): + _require_covered( + "orderbook-rs", + [ + "integrations/flash_benchmark/bridge.py", + "integrations/flash_benchmark/artifacts/orderbook-rs-issue-88-divergence.json", + ], + ) + + +def test_path_glob_matching_keeps_single_star_inside_one_directory(): + assert _matches("src/tracebook/core/order.py", "src/tracebook/core/**") + assert _matches("tests/test_book_replay_campaign.py", "tests/test_book_replay*.py") + assert not _matches("tests/nested/test_book_replay.py", "tests/*.py") diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index fb4b6b8..d1b85ee 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -207,13 +207,26 @@ def test_sdist_excludes_private_experiments_and_local_navigation_material(): ): assert directive in manifest - for path in ("Makefile", ".github/workflows/ci.yml", ".github/workflows/release.yml"): + for path in ( + "Makefile", + ".github/workflows/ci.yml", + ".github/workflows/release.yml", + "CONTRIBUTING.md", + "docs/release.md", + ): workflow = (ROOT / path).read_text(encoding="utf-8") assert ( "tools/verify_distribution_privacy.py dist/conformance/* dist/simulator/*" in workflow ) +def test_local_typecheck_uses_the_active_python_like_ci(): + for path in ("Makefile", "CONTRIBUTING.md", "docs/release.md"): + contents = (ROOT / path).read_text(encoding="utf-8") + assert "mypy --python-version" in contents + assert "sys.version_info[:2]" in contents + + def test_public_install_surfaces_warn_before_the_legacy_ownership_handoff(): install_surfaces = ( ROOT / "README.md", diff --git a/tests/test_release_virtualenvs.py b/tests/test_release_virtualenvs.py new file mode 100644 index 0000000..403981c --- /dev/null +++ b/tests/test_release_virtualenvs.py @@ -0,0 +1,47 @@ +"""Release helpers must preserve the platform defaults of `python -m venv`.""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tools import smoke_conformance_wheel, verify_distribution_split + + +@pytest.mark.parametrize("helper", [smoke_conformance_wheel, verify_distribution_split]) +@pytest.mark.parametrize("platform, use_symlinks", [("posix", True), ("nt", False)]) +def test_release_helpers_preserve_platform_venv_link_mode( + tmp_path, monkeypatch, helper, platform, use_symlinks +): + class EnvironmentRequested(Exception): + pass + + options = {} + destinations = [] + + class RecordingBuilder: + def __init__(self, **kwargs): + options.update(kwargs) + + def create(self, destination): + destinations.append(Path(destination)) + # Stop before installation or qualification: this unit test checks + # the actual builder call, not copies of the configuration text. + raise EnvironmentRequested + + monkeypatch.setattr(helper, "os", SimpleNamespace(name=platform)) + monkeypatch.setattr(helper.venv, "EnvBuilder", RecordingBuilder) + + with pytest.raises(EnvironmentRequested): + if helper is verify_distribution_split: + helper._new_environment(tmp_path, "environment") + else: + wheel = tmp_path / "example.whl" + adapter = tmp_path / "adapter.py" + wheel.write_bytes(b"not installed by this test") + adapter.write_bytes(b"not executed by this test") + helper.main([str(wheel), "--adapter", str(adapter)]) + + assert options == {"with_pip": True, "clear": True, "symlinks": use_symlinks} + assert len(destinations) == 1 + assert not destinations[0].exists() diff --git a/tools/smoke_conformance_wheel.py b/tools/smoke_conformance_wheel.py index 323e993..b58e690 100644 --- a/tools/smoke_conformance_wheel.py +++ b/tools/smoke_conformance_wheel.py @@ -141,7 +141,9 @@ def main(argv: Sequence[str] | None = None) -> int: root = Path(temporary) environment = root / "venv" output = root / "qualification" - venv.EnvBuilder(with_pip=True, clear=True).create(environment) + # Match `python -m venv`: keep executable-relative runtime libraries + # reachable on POSIX, while retaining the default Windows copy mode. + venv.EnvBuilder(with_pip=True, clear=True, symlinks=os.name != "nt").create(environment) python = _environment_python(environment) conformance = _environment_script(environment, "tracebook-conformance") diff --git a/tools/verify_distribution_split.py b/tools/verify_distribution_split.py index ddb503e..764a5f7 100644 --- a/tools/verify_distribution_split.py +++ b/tools/verify_distribution_split.py @@ -574,7 +574,9 @@ def validate_legacy_wheel( def _new_environment(root: Path, name: str) -> tuple[Path, Path]: environment = root / name - venv.EnvBuilder(with_pip=True, clear=True).create(environment) + # Match `python -m venv`: copying a POSIX executable can break its + # executable-relative libpython/framework loader paths (notably on macOS). + venv.EnvBuilder(with_pip=True, clear=True, symlinks=os.name != "nt").create(environment) return environment, _environment_python(environment) From e65416a3b109e320f2686c77106b10403f9e7fad Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Fri, 4 Sep 2026 22:39:15 -0400 Subject: [PATCH 16/19] Fix release-code review findings; preserve frozen research --- CHANGELOG.md | 5 + docs/book-replay-conformance.md | 10 +- docs/pr-84-review-disposition.md | 94 +++++++++++ docs/release.md | 2 + integrations/gocronx_matcher/README.md | 3 + integrations/gocronx_matcher/src/adapter.rs | 35 +++- integrations/intrepid_orderbook/README.md | 2 +- integrations/intrepid_orderbook/adapter.go | 2 +- .../intrepid_orderbook/adapter_test.go | 47 ++++++ integrations/intrepid_orderbook/protocol.go | 11 +- integrations/orderbook_rs/README.md | 10 +- integrations/orderbook_rs/src/adapter.rs | 39 ++++- integrations/rust_protocol/src/quantity.rs | 40 ++++- src/tracebook/book_replay/artifacts.py | 72 +++++++-- src/tracebook/book_replay/campaign.py | 2 +- src/tracebook/book_replay/cli.py | 5 +- src/tracebook/book_replay/external.py | 16 +- src/tracebook/book_replay/model.py | 16 ++ src/tracebook/book_replay/protocol.py | 73 ++++++--- src/tracebook/book_replay/reference.py | 12 +- src/tracebook/conformance/model.py | 5 +- tests/fixtures/faulty_book_replay_adapter.py | 4 +- tests/test_book_replay.py | 153 ++++++++++++++++++ tests/test_book_replay_artifacts.py | 77 ++++++++- tests/test_book_replay_campaign.py | 47 ++---- tests/test_distribution_privacy.py | 19 +++ tests/test_integration_workflow_coverage.py | 77 ++++++++- tools/verify_distribution_privacy.py | 3 +- 28 files changed, 761 insertions(+), 120 deletions(-) create mode 100644 docs/pr-84-review-disposition.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 92d1ef5..63a9d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions. ## Unreleased +- Addressed the release-code findings from PR #84: fail closed on L3 shutdown + and malformed protocol values, preserve exact decimal probe arithmetic, + protect staged artifacts against ordinary source-path replacement, reject + zero-normalized Rust observations, and enforce Go IOC/FOK numeric limits. + Frozen research findings remain separately documented and unresolved. - Kept Rust adapter input quantities independent of report display precision. Both native adapters now share exact checked 12-place integer encoding and reject quantities outside that domain rather than pre-rounding orders. diff --git a/docs/book-replay-conformance.md b/docs/book-replay-conformance.md index 5f07a2f..59d46bc 100644 --- a/docs/book-replay-conformance.md +++ b/docs/book-replay-conformance.md @@ -112,12 +112,20 @@ files created by that attempt. These are individual file commits, not a filesystem-wide transaction: abrupt process termination can leave a complete trace without its report. Only use the pair after the command completes. Sibling `.NAME.tracebook-in-progress` reservations and `.tracebook-stage-*` -temporary files are removed on normal completion, with cleanup attempted on +private staging directories are removed on normal completion, with cleanup attempted on handled errors. Filesystem cleanup failures or competing replacements can leave sidecars. After a forced stop or cleanup failure, retain the artifacts and use fresh output names for a new invocation; do not remove a live process's reservations. +Staged payloads live in owner-only, read-only directories and publication uses +the open directory descriptor, so replacing a staging directory's parent entry +cannot substitute another payload. Published files are owner-readable. This +protects against ordinary concurrent file writers, not privileged processes or +same-user code deliberately changing permissions or writing through an already +open descriptor. Use a trusted output directory; these filesystem guards are +not a sandbox for hostile code running as your user. + ## Generated Campaigns And Reduction The v1 generator uses a specified SplitMix64 stream rather than Python's diff --git a/docs/pr-84-review-disposition.md b/docs/pr-84-review-disposition.md new file mode 100644 index 0000000..0547b6a --- /dev/null +++ b/docs/pr-84-review-disposition.md @@ -0,0 +1,94 @@ +# PR #84 Review Disposition + +This records the 30 comments in the initial automated review of +[PR #84](https://github.com/Taz33m/tracebook/pull/84), against `0c98e18`. +It is a disposition of that review, not release approval or a new experiment +result. Source and regression tests, not the review's severity labels, determine +the implemented changes. + +## Release-code fixes + +The following 17 comments are addressed in the release implementation. Comment +IDs identify the original GitHub review comments. Existing qualified-profile +identities and retained evidence are not rewritten to accommodate a change. + +| Comment ID | Change | Regression / verification surface | +| --- | --- | --- | +| 3939088988 | Close the L3 adapter before emitting `complete`; a failed close is one terminal adapter error, with no retry. | `test_server_close_failure_is_one_terminal_adapter_error` | +| 3939088991 | Both Rust adapters fail before emitting a positive fill/remainder normalized to zero. Inputs remain exact native lots. | Shared quantity tests and both adapters' `positive_fills_and_remainders_cannot_be_emitted_as_zero` | +| 3939089005 | Share the faulty FIFO-reordering fixture between in-process and external tests; retain each existing engine identity. | Existing campaign, reduction, and failure-ID regressions | +| 3939089008 | Validate typed L3 outcomes, fills, snapshot entries, and book collections during construction. | `test_invalid_in_process_observation_is_classified_as_protocol_failure` | +| 3939089018 | Correct the orderbook-rs translation contract: 12 native decimal places, independent output precision. | Quantity conversion regressions and README review | +| 3939089021 | Exact FIFO probe subtraction independent of ambient Decimal precision; remove context-rounding from shared canonical formatting too. | `test_probe_subtraction_and_canonical_values_are_exact_at_any_context_precision` | +| 3939089023 | Separate client decoding/validation failures from exceptions raised inside adapters. | Factory/apply/snapshot exception tests and malformed-payload test | +| 3939089028 | Require actual non-negative integer snapshot/finish counters; reject booleans, missing values, null, floats, and strings. | Parameterized session-counter tests | +| 3939089029 | Require Go snapshot/finish counters and an object event payload, without zero-value defaults accepting missing/null fields. | `TestServerRejectsMissingNullAndMistypedRequiredFields` | +| 3939089032 | Normalize timeout conversion errors, including oversized integers, into the public L3 error contract. | `test_external_timeout_overflow_is_a_public_validation_error` | +| 3939089038 | Wrap invalid ready-frame engine metadata as a book-replay protocol error and reap the child. | Invalid-metadata/child-reaping regression | +| 3939089052 | Stage in a private read-only directory and link relative to its open descriptor. Protect the payload entry from ordinary competing writers and anchor publication across directory replacement. | Stage-entry and stage-directory replacement tests, plus existing rollback/competing-destination tests | +| 3939089055 | Count price relocation only when the price actually changes, not on side-only updates. | `test_side_only_update_does_not_claim_price_relocation_coverage` | +| 3939089057 | Apply the documented conservative same-side aggregate envelope to IOC/FOK as well as GTC limit orders; retain the market exemption. | `TestSideAggregateEnvelopeIncludesIOCAndFOKButNotMarket` | +| 3939089063 | Report symlink-loop path resolution as a controlled CLI error before starting a candidate. | Symlink-loop CLI regression | +| 3939089067 | Handle corrupt deflated source inventories as a failed distribution privacy check. | Corrupt real ZIP-block regression | +| 3939089071 | Bound the workflow path reader to `on.pull_request.paths`, allowing comments, blank lines, different indentation, and other trigger keys. | Positive layout variants and negative job/other-trigger regressions | + +Two limits matter when interpreting these fixes: + +- Insufficient output precision is an operational error, not a semantic + rejection or a successful qualification. Native matching inputs are not + rounded, inflated, retried, or silently removed. The Python CLOB reference + also rejects zero normalized positive quantities; its positive-value wire + contract has not been relaxed. +- Artifact publication is not a security sandbox against root or same-user + code deliberately changing permissions or writing through existing file + descriptors. Use trusted output directories. Normal concurrent writers must + not overwrite destinations, and rollback preserves competing files. The + directory-permission test is intentionally skipped when running as root. + +## Frozen research findings — open, not silently patched + +These 13 findings are in the historical/hash-bound research harness. The +accepted scope leaves that code, frozen inputs, evaluator material, and retained +results unchanged. They remain open pending a separate research disposition; +the release fixes above do not resolve them. + +| Comment ID | Reported issue | Required next decision/check | +| --- | --- | --- | +| 3939088982 | V2 Claude auth-route probe inherits ambient environment. | Audit route-binding evidence locally; use a sanitized, explicit environment in a newly versioned runner. | +| 3939088984 | V2 finalization failure before verdict can quarantine a completed provider turn. | Audit lifecycle receipts without rerunning providers; a future runner needs a durable provider-completion barrier before finalization. | +| 3939088987 | V2 frozen snapshots permit symbolic links. | Assess snapshot inventories and provider read boundaries locally; preregister an explicit link policy for future runs. | +| 3939088994 | Baseline live copy is not checked against its frozen source before launch. | Audit existing snapshot bindings; bind and verify copied workspaces in a future runner. | +| 3939089000 | Baseline command success can outlive a failed/timed-out provider. | Audit lifecycle exit receipts; separate transport completion and semantic outcomes in a new runner. | +| 3939089002 | Treatment technical verdict handling exempts some unsuccessful provider exits/timeouts. | Audit treatment lifecycle verdicts separately from V2, which has its own completion checks. | +| 3939089010 | Missing/unreadable V2 bound file can escape as `OSError`. | Add controlled validation errors in a future version, preserving frozen hash bindings. | +| 3939089043 | Baseline missing reported model falls back to `opus`. | Treat absent model evidence as missing, not observed identity, in a separately versioned audit/fix. | +| 3939089045 | Baseline permits an execution-time timeout override. | Compare receipts to the preregistered budget before deciding whether existing runs are affected. | +| 3939089048 | Baseline candidate/gold paths lack private-root confinement. | Audit declared paths locally; enforce resolved boundaries in a new runner. | +| 3939089050 | Baseline case IDs admit path components. | Audit manifest IDs; enforce a safe identifier grammar in a new runner. | +| 3939089056 | Treatment validation omits some workspace evidence artifacts. | Audit artifact completeness locally and specify required evidence in a new validator version. | +| 3939089060 | Baseline quarantine checks a verdict sentinel that baseline does not write. | Distinguish completed turns from incomplete interruptions using lifecycle receipts, not a missing sentinel. | + +These are source-level risks, not proof that a particular completed cohort was +contaminated. Do not pool different harness generations, modify an old frozen +runner in place, reinterpret missing metadata as observed evidence, or rerun a +completed provider turn. An approved audit should be local and preserve every +interruption artifact; any necessary future collection needs a new versioned +protocol and explicit authorization. No private gold or evaluator artifacts +belong in a provider request or this public document. + +## Release gate + +Local verification of the fixes passed: Python 3.13 had 651 passing tests, +one skip, and 82.38% coverage; the focused Python 3.10 regressions passed too. +Formatting, lint, type checking, static security checks, pinned Rust 1.88/Go +1.23.5 checks, distribution privacy, and sdist/wheel agreement passed. +Fresh 5,000-event qualifications retained the exact orderbook-rs, gocronx, Go +FIFO-limit, and Go partial-fill identities. The Go full-profile FOK failure and +the Nautilus 2,500-event positive campaign/three-event negative control retained +their expected identities and reduced trace. These are local checks; consult +the PR for the latest remote CI and review state. + +Keep the research conversations open until their disposition is agreed. Passing +release tests does not justify dismissing those comments, bypassing repository +protections, merging the PR, or publishing a tag. The previous maintainer +invitation and its immutable source/wheel references remain unchanged. diff --git a/docs/release.md b/docs/release.md index 35d7bfb..416fe05 100644 --- a/docs/release.md +++ b/docs/release.md @@ -65,6 +65,8 @@ python -m pip check ## Remote Verification +- Review the [PR #84 disposition](pr-84-review-disposition.md): release-code + fixes do not close the separately frozen research findings or authorize a release. - Push to a branch and wait for Ubuntu GitHub Actions on Python 3.10 through 3.13. - Confirm the native `orderbook-rs` integration passes its fixed trace, `7/9` diff --git a/integrations/gocronx_matcher/README.md b/integrations/gocronx_matcher/README.md index 8cc1ed1..f36838d 100644 --- a/integrations/gocronx_matcher/README.md +++ b/integrations/gocronx_matcher/README.md @@ -50,6 +50,9 @@ traces, 10/10 covered capabilities, and 28 candidate process runs. The adapter scales prices into integer ticks and quantities into fixed twelve-place native lots. `quantity_decimal_places` controls output rounding only; it never rounds submitted order, reduction, or replacement quantities. +If normalization would turn a positive fill or remainder into zero, the session +fails with an adapter error before emitting that invalid observation. Use enough +output precision for the trace; this does not round or resubmit native orders. For example, at output precision zero, crossing a `2.4` buy with a `3.5` sell leaves native quantity `1.1`, reported as `1`. Nonrepresentable quantities are rejected before submission, and invalid replacement quantities preserve the diff --git a/integrations/gocronx_matcher/src/adapter.rs b/integrations/gocronx_matcher/src/adapter.rs index 86ec5e8..0b0d39b 100644 --- a/integrations/gocronx_matcher/src/adapter.rs +++ b/integrations/gocronx_matcher/src/adapter.rs @@ -75,7 +75,7 @@ impl AdapterConfig { Ok(canonical_decimal(price)) } - fn format_quantity(&self, units: u64) -> String { + fn format_quantity(&self, units: u64) -> Result { self.quantity_encoding.format(units) } } @@ -389,7 +389,7 @@ impl Adapter { buy_order_id: trade.buy_id.get(), sell_order_id: trade.sell_id.get(), price: self.config.format_price(trade.price.get())?, - quantity: self.config.format_quantity(trade.quantity.get()), + quantity: self.config.format_quantity(trade.quantity.get())?, }) } @@ -432,7 +432,7 @@ impl Adapter { Ok(RestingOrder { order_id: order.order_id, price: self.config.format_price(order.price)?, - remaining_quantity: self.config.format_quantity(order.remaining), + remaining_quantity: self.config.format_quantity(order.remaining)?, owner, order_type: "LIMIT", }) @@ -617,6 +617,35 @@ mod tests { assert_eq!(state.books[0].asks[0].remaining_quantity, "1"); } + #[test] + fn positive_fills_and_remainders_cannot_be_emitted_as_zero() { + for (maker, taker) in [(0.4, None), (1.0, Some(0.4)), (1.0, Some(0.6))] { + let mut adapter = Adapter::new(ConfigWire { + quantity_decimal_places: 0, + ..ConfigWire::default() + }) + .unwrap(); + let first = serde_json::from_value::(serde_json::json!({ + "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":maker + })) + .unwrap(); + let result = if let Some(quantity) = taker { + adapter.apply(&first, 1).unwrap(); + let second = serde_json::from_value::(serde_json::json!({ + "op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":quantity + })).unwrap(); + adapter.apply(&second, 2) + } else { + adapter.apply(&first, 1) + }; + assert!( + result + .unwrap_err() + .contains("positive quantity cannot be represented") + ); + } + } + #[test] fn output_precision_does_not_round_reduce_or_replace_inputs() { let mut adapter = Adapter::new(ConfigWire { diff --git a/integrations/intrepid_orderbook/README.md b/integrations/intrepid_orderbook/README.md index 03f28ea..f2d9f83 100644 --- a/integrations/intrepid_orderbook/README.md +++ b/integrations/intrepid_orderbook/README.md @@ -42,7 +42,7 @@ precision is lower. The native arithmetic has a bounded domain: - price ticks and scaled quantities must be positive and fit signed `int64`; -- before admitting a limit order, the adapter requires its full quantity plus +- before admitting a limit, IOC, or FOK order, the adapter requires its full quantity plus all resting quantity on its side to fit signed `int64` (replacement excludes the old order); - under pro-rata, incoming quantity times each potentially crossed maker's diff --git a/integrations/intrepid_orderbook/adapter.go b/integrations/intrepid_orderbook/adapter.go index 51a642a..7de94e0 100644 --- a/integrations/intrepid_orderbook/adapter.go +++ b/integrations/intrepid_orderbook/adapter.go @@ -252,7 +252,7 @@ func (book *bookHarness) checkQuantityCapacity( return errors.New("quantity exceeds the native int64 pro-rata product range") } } - if orderType == "LIMIT" && quantity > math.MaxInt64-sameSideTotal { + if orderType != "MARKET" && quantity > math.MaxInt64-sameSideTotal { return errors.New("quantity exceeds the native int64 side aggregate range") } return nil diff --git a/integrations/intrepid_orderbook/adapter_test.go b/integrations/intrepid_orderbook/adapter_test.go index b59400e..7c49251 100644 --- a/integrations/intrepid_orderbook/adapter_test.go +++ b/integrations/intrepid_orderbook/adapter_test.go @@ -173,6 +173,53 @@ func TestServerHandshakeAndEmptyCompletion(t *testing.T) { } } +func TestServerRejectsMissingNullAndMistypedRequiredFields(t *testing.T) { + for _, frame := range []string{ + `{"type":"finish"}`, `{"type":"finish","event_count":null}`, + `{"type":"finish","event_count":false}`, `{"type":"finish","event_count":"0"}`, + `{"type":"finish","event_count":0.0}`, `{"type":"finish","event_count":-1}`, + `{"type":"snapshot"}`, `{"type":"snapshot","index":null}`, + `{"type":"snapshot","index":true}`, `{"type":"snapshot","index":"0"}`, + `{"type":"event","index":1}`, `{"type":"event","index":1,"event":null}`, + `{"type":"event","index":1,"event":[]}`, `{"type":"event","index":1,"event":false}`, + } { + t.Run(frame, func(t *testing.T) { + input := `{"type":"hello","protocol":"tracebook.conformance","protocol_version":1}` + "\n" + frame + var output bytes.Buffer + if status := serve(strings.NewReader(input), &output); status != 2 { + t.Fatalf("status = %d, output = %s", status, output.String()) + } + lines := strings.Split(strings.TrimSpace(output.String()), "\n") + if len(lines) != 2 || !strings.Contains(lines[1], `"code":"PROTOCOL_ERROR"`) { + t.Fatalf("expected ready and one protocol error: %s", output.String()) + } + }) + } +} + +func TestSideAggregateEnvelopeIncludesIOCAndFOKButNotMarket(t *testing.T) { + for _, orderType := range []string{"LIMIT", "IOC", "FOK", "MARKET"} { + t.Run(orderType, func(t *testing.T) { + harness := testAdapter(t, configWire{}) + first := applyForTest(t, harness, limitEvent("1", "BUY", "0.01", "4700000000000")) + event := limitEvent("2", "BUY", "0.01", "4700000000000") + event.OrderType = orderType + got := applyForTest(t, harness, event) + if got.StateHash != first.StateHash || len(got.Trades) != 0 { + t.Fatalf("unsupported aggregate mutated the book: %#v", got) + } + if orderType == "MARKET" { + if got.Outcome.Status != "applied" { + t.Fatalf("market order lost its aggregate exemption: %#v", got) + } + } else if got.Outcome.Status != "rejected" || got.Outcome.Message == nil || + !strings.Contains(*got.Outcome.Message, "side aggregate range") { + t.Fatalf("expected pre-mutation aggregate rejection: %#v", got) + } + }) + } +} + func TestServerHandshakeReportsInjectedEvidenceIdentity(t *testing.T) { previousRevision, previousSnapshot := engineRevision, engineSnapshot t.Cleanup(func() { diff --git a/integrations/intrepid_orderbook/protocol.go b/integrations/intrepid_orderbook/protocol.go index 6c08c55..b8f66a6 100644 --- a/integrations/intrepid_orderbook/protocol.go +++ b/integrations/intrepid_orderbook/protocol.go @@ -304,6 +304,9 @@ func runServer(decoder *json.Decoder, writer *bufio.Writer) error { } switch kind { case "event": + if raw := object["event"]; len(raw) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return protocolError("event payload must be an object") + } var frame eventFrame if err := decodeObject(object, &frame); err != nil { return protocolError(err.Error()) @@ -321,12 +324,12 @@ func runServer(decoder *json.Decoder, writer *bufio.Writer) error { } case "snapshot": var frame struct { - Index uint64 `json:"index"` + Index *uint64 `json:"index"` } if err := decodeObject(object, &frame); err != nil { return protocolError(err.Error()) } - if frame.Index != lastIndex { + if frame.Index == nil || *frame.Index != lastIndex { return protocolError("snapshot index does not match the last event") } state, err := adapter.snapshot() @@ -340,12 +343,12 @@ func runServer(decoder *json.Decoder, writer *bufio.Writer) error { } case "finish": var frame struct { - EventCount uint64 `json:"event_count"` + EventCount *uint64 `json:"event_count"` } if err := decodeObject(object, &frame); err != nil { return protocolError(err.Error()) } - if frame.EventCount != lastIndex { + if frame.EventCount == nil || *frame.EventCount != lastIndex { return protocolError("finish event_count does not match the last event") } return writeFrame(writer, map[string]any{ diff --git a/integrations/orderbook_rs/README.md b/integrations/orderbook_rs/README.md index 323ecdd..5e4329d 100644 --- a/integrations/orderbook_rs/README.md +++ b/integrations/orderbook_rs/README.md @@ -230,8 +230,14 @@ artifacts under `target/` are excluded. rounding, are stored as integer ticks, and return as canonical decimal strings. This is intentionally not exact decimal division at half-tick boundaries; for example, `1.015 / 0.01` snaps to `1.01`. -- Quantities use fixed-point `u64` units at `quantity_decimal_places`; a value - that rounds to zero or overflows that range is rejected. +- Quantities use fixed-point `u64` units at 12 native decimal places. Inputs + requiring finer precision or exceeding that native range are rejected without + rounding. `quantity_decimal_places` controls output normalization only; it + never changes the quantity submitted to the engine. + If output normalization would turn a positive fill or remainder into zero, + the session fails with an adapter error before emitting an invalid observation. + Choose enough output precision for the trace; low-precision configurations + outside the recorded profiles are not qualified by those results. - Real owners map deterministically to `Hash32`. Anonymous owners receive a unique per-order identity so STP does not make unrelated anonymous orders self-match or reject them for a missing user ID. diff --git a/integrations/orderbook_rs/src/adapter.rs b/integrations/orderbook_rs/src/adapter.rs index b1b0c1e..c2e15ae 100644 --- a/integrations/orderbook_rs/src/adapter.rs +++ b/integrations/orderbook_rs/src/adapter.rs @@ -114,7 +114,7 @@ impl AdapterConfig { Ok(canonical_decimal(price)) } - fn format_quantity(&self, units: u64) -> String { + fn format_quantity(&self, units: u64) -> Result { self.quantity_encoding.format(units) } } @@ -614,7 +614,7 @@ impl Adapter { buy_order_id, sell_order_id, price: self.config.format_price(trade.price().as_u128())?, - quantity: self.config.format_quantity(trade.quantity().as_u64()), + quantity: self.config.format_quantity(trade.quantity().as_u64())?, }); } } @@ -655,7 +655,7 @@ impl Adapter { Ok(RestingOrder { order_id, price: self.config.format_price(order.price().as_u128())?, - remaining_quantity: self.config.format_quantity(visible_quantity(order)), + remaining_quantity: self.config.format_quantity(visible_quantity(order))?, owner, order_type: "LIMIT", }) @@ -779,8 +779,8 @@ mod tests { let high = serde_json::Number::from_str("1.2355").unwrap(); assert_eq!(config.quantity_units(&low).unwrap(), 1_234_500_000_000); assert_eq!(config.quantity_units(&high).unwrap(), 1_235_500_000_000); - assert_eq!(config.format_quantity(1_234_500_000_000), "1.234"); - assert_eq!(config.format_quantity(1_235_500_000_000), "1.236"); + assert_eq!(config.format_quantity(1_234_500_000_000).unwrap(), "1.234"); + assert_eq!(config.format_quantity(1_235_500_000_000).unwrap(), "1.236"); } #[test] @@ -813,6 +813,35 @@ mod tests { assert_eq!(state.books[0].asks[0].remaining_quantity, "1"); } + #[test] + fn positive_fills_and_remainders_cannot_be_emitted_as_zero() { + for (maker, taker) in [(0.4, None), (1.0, Some(0.4)), (1.0, Some(0.6))] { + let mut adapter = Adapter::new(ConfigWire { + quantity_decimal_places: 0, + ..ConfigWire::default() + }) + .unwrap(); + let first = serde_json::from_value::(serde_json::json!({ + "op":"new","symbol":"TEST","order_id":1,"side":"BUY","price":100,"quantity":maker + })) + .unwrap(); + let result = if let Some(quantity) = taker { + adapter.apply(&first, 1).unwrap(); + let second = serde_json::from_value::(serde_json::json!({ + "op":"new","symbol":"TEST","order_id":2,"side":"SELL","price":100,"quantity":quantity + })).unwrap(); + adapter.apply(&second, 2) + } else { + adapter.apply(&first, 1) + }; + assert!( + result + .unwrap_err() + .contains("positive quantity cannot be represented") + ); + } + } + #[test] fn output_precision_does_not_round_reduce_or_replace_inputs() { let mut adapter = Adapter::new(ConfigWire { diff --git a/integrations/rust_protocol/src/quantity.rs b/integrations/rust_protocol/src/quantity.rs index ead2ae1..41009b8 100644 --- a/integrations/rust_protocol/src/quantity.rs +++ b/integrations/rust_protocol/src/quantity.rs @@ -60,7 +60,7 @@ impl QuantityEncoding { coefficient.checked_mul(multiplier).ok_or_else(range_error) } - pub fn format(&self, units: u64) -> String { + pub fn format(&self, units: u64) -> Result { let places = self.output_decimal_places.min(NATIVE_DECIMAL_PLACES); let divisor = 10_u64.pow(NATIVE_DECIMAL_PLACES - places); let mut rounded = units / divisor; @@ -71,15 +71,23 @@ impl QuantityEncoding { // With divisor >= 10, incrementing the quotient cannot overflow. rounded += 1; } + if rounded == 0 { + return Err(format!( + "positive quantity cannot be represented at quantity_decimal_places={}", + self.output_decimal_places + )); + } let scale = 10_u64.pow(places); let whole = rounded / scale; let fraction = rounded % scale; if fraction == 0 { - return whole.to_string(); + return Ok(whole.to_string()); } - format!("{whole}.{fraction:0width$}", width = places as usize) - .trim_end_matches('0') - .to_string() + Ok( + format!("{whole}.{fraction:0width$}", width = places as usize) + .trim_end_matches('0') + .to_string(), + ) } } @@ -155,7 +163,7 @@ mod tests { ] { assert!(encoding.units(&number(text)).is_err(), "{text}"); } - assert_eq!(encoding.format(u64::MAX), "18446744.073709551615"); + assert_eq!(encoding.format(u64::MAX).unwrap(), "18446744.073709551615"); } #[test] @@ -170,10 +178,28 @@ mod tests { ] { let encoding = QuantityEncoding::new(places).unwrap(); assert_eq!( - encoding.format(encoding.units(&number(text)).unwrap()), + encoding + .format(encoding.units(&number(text)).unwrap()) + .unwrap(), expected ); } assert!(QuantityEncoding::new(19).is_err()); } + + #[test] + fn unrepresentable_positive_observations_fail_instead_of_emitting_zero() { + for (places, text) in [(0, "0.4"), (0, "0.5"), (3, "0.0004"), (11, "1e-12")] { + let encoding = QuantityEncoding::new(places).unwrap(); + let units = encoding.units(&number(text)).unwrap(); + assert!(units > 0); + assert!( + encoding + .format(units) + .unwrap_err() + .contains("cannot be represented") + ); + } + assert!(QuantityEncoding::new(12).unwrap().format(0).is_err()); + } } diff --git a/src/tracebook/book_replay/artifacts.py b/src/tracebook/book_replay/artifacts.py index f762a14..12f2eea 100644 --- a/src/tracebook/book_replay/artifacts.py +++ b/src/tracebook/book_replay/artifacts.py @@ -28,10 +28,15 @@ class OutputReservation: def __init__(self, destination: str | Path) -> None: path = Path(destination).expanduser() self._requested_target = path.absolute() - self.target = path.parent.resolve() / path.name + try: + self.target = path.parent.resolve() / path.name + except RuntimeError as exc: + raise BookReplayError(f"cannot resolve book-replay path: {exc}") from exc self._lock_name = f".{self.target.name}.tracebook-in-progress" - self._stage_name = f".tracebook-stage-{uuid.uuid4().hex}" + self._stage_directory_name = f".tracebook-stage-{uuid.uuid4().hex}" + self._stage_name = "payload" self._parent_fd: Optional[int] = None + self._stage_directory_fd: Optional[int] = None self._lock_fd: Optional[int] = None self._stage_fd: Optional[int] = None self._published = False @@ -50,7 +55,21 @@ def __enter__(self) -> "OutputReservation": self._require_absent() flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) self._lock_fd = os.open(self._lock_name, flags, 0o600, dir_fd=self._parent_fd) - self._stage_fd = os.open(self._stage_name, flags, 0o600, dir_fd=self._parent_fd) + os.mkdir(self._stage_directory_name, mode=0o700, dir_fd=self._parent_fd) + self._stage_directory_fd = os.open( + self._stage_directory_name, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=self._parent_fd, + ) + self._stage_fd = os.open( + self._stage_name, flags, 0o400, dir_fd=self._stage_directory_fd + ) + # Keep the source entry in a private, non-writable directory. The + # write descriptor remains usable, but another normal file writer + # cannot replace the source entry in the validate/link window. + # Linking relative to this directory's descriptor also survives a + # competing rename/replacement of its entry in the output parent. + os.fchmod(self._stage_directory_fd, 0o500) self.validate() except BaseException: try: @@ -74,13 +93,16 @@ def _require_absent(self) -> None: return raise BookReplayError(f"book-replay output already exists: {self.target}") - def _owns_entry(self, name: str, descriptor: Optional[int]) -> bool: - if self._parent_fd is None or descriptor is None: + def _owns_entry( + self, name: str, descriptor: Optional[int], *, directory: Optional[int] = None + ) -> bool: + directory = self._parent_fd if directory is None else directory + if directory is None or descriptor is None: return False try: - return _identity( - os.stat(name, dir_fd=self._parent_fd, follow_symlinks=False) - ) == _identity(os.fstat(descriptor)) + return _identity(os.stat(name, dir_fd=directory, follow_symlinks=False)) == _identity( + os.fstat(descriptor) + ) except OSError: return False @@ -90,7 +112,10 @@ def validate(self) -> None: if ( not self._same_parent() or not self._owns_entry(self._lock_name, self._lock_fd) - or not self._owns_entry(self._stage_name, self._stage_fd) + or not self._owns_entry(self._stage_directory_name, self._stage_directory_fd) + or not self._owns_entry( + self._stage_name, self._stage_fd, directory=self._stage_directory_fd + ) ): raise BookReplayError(f"book-replay output reservation changed: {self.target}") self._require_absent() @@ -123,7 +148,7 @@ def publish(self) -> None: os.link( self._stage_name, self.target.name, - src_dir_fd=self._parent_fd, + src_dir_fd=self._stage_directory_fd, dst_dir_fd=self._parent_fd, follow_symlinks=False, ) @@ -144,14 +169,19 @@ def rollback(self) -> None: def close(self) -> None: error: Optional[OSError] = None - for name, descriptor in ( - (self._stage_name, self._stage_fd), - (self._lock_name, self._lock_fd), + if self._stage_directory_fd is not None: + try: + os.fchmod(self._stage_directory_fd, 0o700) + except OSError as exc: + error = exc + for name, descriptor, directory in ( + (self._stage_name, self._stage_fd, self._stage_directory_fd), + (self._lock_name, self._lock_fd, self._parent_fd), ): if descriptor is not None: try: - if self._owns_entry(name, descriptor): - os.unlink(name, dir_fd=self._parent_fd) + if self._owns_entry(name, descriptor, directory=directory): + os.unlink(name, dir_fd=directory) except OSError as exc: error = error or exc finally: @@ -160,6 +190,18 @@ def close(self) -> None: except OSError as exc: error = error or exc self._stage_fd = self._lock_fd = None + if self._stage_directory_fd is not None: + try: + if self._owns_entry(self._stage_directory_name, self._stage_directory_fd): + os.rmdir(self._stage_directory_name, dir_fd=self._parent_fd) + except OSError as exc: + error = error or exc + finally: + try: + os.close(self._stage_directory_fd) + except OSError as exc: + error = error or exc + self._stage_directory_fd = None if self._parent_fd is not None: try: os.close(self._parent_fd) diff --git a/src/tracebook/book_replay/campaign.py b/src/tracebook/book_replay/campaign.py index c696658..f860485 100644 --- a/src/tracebook/book_replay/campaign.py +++ b/src/tracebook/book_replay/campaign.py @@ -307,7 +307,7 @@ def measure_book_replay_coverage( if previous.side == event.side and previous.price == event.price: if Decimal(event.quantity or "0") > Decimal(previous.quantity or "0"): covered.add("same-level-upsize") - else: + elif previous.price != event.price: covered.add("price-relocation") active[key] = event elif event.op == "delete": diff --git a/src/tracebook/book_replay/cli.py b/src/tracebook/book_replay/cli.py index 2b4ab45..60ae764 100644 --- a/src/tracebook/book_replay/cli.py +++ b/src/tracebook/book_replay/cli.py @@ -125,7 +125,10 @@ def _emit( def _require_distinct_paths(*paths: Optional[str]) -> None: - resolved = [Path(path).expanduser().resolve() for path in paths if path is not None] + try: + resolved = [Path(path).expanduser().resolve() for path in paths if path is not None] + except RuntimeError as exc: + raise BookReplayError(f"cannot resolve book-replay path: {exc}") from exc if len(resolved) != len(set(resolved)) or any( left in right.parents or right in left.parents for index, left in enumerate(resolved) diff --git a/src/tracebook/book_replay/external.py b/src/tracebook/book_replay/external.py index 14101ae..f2a286b 100644 --- a/src/tracebook/book_replay/external.py +++ b/src/tracebook/book_replay/external.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Deque, Mapping, Optional, Sequence +from ..conformance.model import ConformanceError from .model import ( PROTOCOL_NAME, PROTOCOL_VERSION, @@ -46,16 +47,20 @@ def __init__( if not isinstance(argument, str) or not argument: raise BookReplayError("candidate command arguments must be non-empty strings") normalized_command.append(argument) + try: + normalized_timeout = float(timeout_seconds) + except (TypeError, ValueError, OverflowError) as exc: + raise BookReplayError("timeout_seconds must be a positive finite number") from exc if ( isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, (int, float)) - or not math.isfinite(float(timeout_seconds)) - or timeout_seconds <= 0 + or not math.isfinite(normalized_timeout) + or normalized_timeout <= 0 ): raise BookReplayError("timeout_seconds must be a positive finite number") self.command = tuple(normalized_command) - self.timeout_seconds = float(timeout_seconds) + self.timeout_seconds = normalized_timeout self._closed = False self._broken = False self._last_index = 0 @@ -112,7 +117,10 @@ def __init__( raise BookReplayProtocolError("ready message reported the wrong protocol") if ready.get("protocol_version") != PROTOCOL_VERSION: raise BookReplayProtocolError("ready message reported the wrong protocol version") - self.metadata = EngineMetadata.from_dict(ready.get("engine", {})) + try: + self.metadata = EngineMetadata.from_dict(ready.get("engine", {})) + except ConformanceError as exc: + raise BookReplayProtocolError(f"invalid ready engine metadata: {exc}") from exc except Exception: self._broken = True self._shutdown() diff --git a/src/tracebook/book_replay/model.py b/src/tracebook/book_replay/model.py index 96c0ef9..57b42c5 100644 --- a/src/tracebook/book_replay/model.py +++ b/src/tracebook/book_replay/model.py @@ -213,6 +213,12 @@ class BookReplaySnapshot: def __post_init__(self) -> None: object.__setattr__(self, "symbol", _symbol(self.symbol)) + for name in ("bids", "asks"): + orders = getattr(self, name) + if not isinstance(orders, tuple) or any( + not isinstance(order, RestingBookOrder) for order in orders + ): + raise BookReplayError(f"{name} must be a tuple of RestingBookOrder values") order_ids = [order.order_id for order in self.bids + self.asks] if len(order_ids) != len(set(order_ids)): raise BookReplayError(f"book {self.symbol!r} contains duplicate order ids") @@ -246,6 +252,10 @@ class BookReplayState: books: Tuple[BookReplaySnapshot, ...] = () def __post_init__(self) -> None: + if not isinstance(self.books, tuple) or any( + not isinstance(book, BookReplaySnapshot) for book in self.books + ): + raise BookReplayError("books must be a tuple of BookReplaySnapshot values") symbols = [book.symbol for book in self.books] if symbols != sorted(symbols) or len(symbols) != len(set(symbols)): raise BookReplayError("book snapshots must have unique symbols in sorted order") @@ -282,6 +292,12 @@ class BookReplayObservation: def __post_init__(self) -> None: object.__setattr__(self, "index", _positive_int(self.index, "observation index")) + if not isinstance(self.outcome, Outcome): + raise BookReplayError("observation outcome must be an Outcome") + if not isinstance(self.fills, tuple) or any( + not isinstance(fill, SimulatedFill) for fill in self.fills + ): + raise BookReplayError("observation fills must be a tuple of SimulatedFill values") if ( not isinstance(self.state_hash, str) or len(self.state_hash) != _HASH_LENGTH diff --git a/src/tracebook/book_replay/protocol.py b/src/tracebook/book_replay/protocol.py index 9ad8dfc..2fa16e4 100644 --- a/src/tracebook/book_replay/protocol.py +++ b/src/tracebook/book_replay/protocol.py @@ -32,6 +32,26 @@ def close(self) -> None: ... BookReplayAdapterFactory = Callable[[BookReplayConfig], BookReplayAdapter] +class _RequestError(ValueError): + """Invalid client input, distinct from exceptions inside an adapter.""" + + +def _message(line: str) -> dict: + try: + message = json.loads(line) + except json.JSONDecodeError as exc: + raise _RequestError(str(exc)) from exc + if not isinstance(message, dict): + raise _RequestError("protocol message must be an object") + return message + + +def _index(value: object, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise _RequestError(f"{name} must be a non-negative integer") + return value + + def _write_message(stream: TextIO, payload: dict) -> None: stream.write(json.dumps(payload, separators=(",", ":"), allow_nan=False) + "\n") stream.flush() @@ -50,15 +70,19 @@ def serve_book_replay_stdio( try: first_line = source.readline() if not first_line: - raise ValueError("expected hello message") - hello = json.loads(first_line) - if not isinstance(hello, dict) or hello.get("type") != "hello": - raise ValueError("first message must be hello") + raise _RequestError("expected hello message") + hello = _message(first_line) + if hello.get("type") != "hello": + raise _RequestError("first message must be hello") if hello.get("protocol") != PROTOCOL_NAME: - raise ValueError(f"protocol must be {PROTOCOL_NAME!r}") - if hello.get("protocol_version") != PROTOCOL_VERSION: - raise ValueError(f"protocol_version must be {PROTOCOL_VERSION}") - adapter = adapter_factory(BookReplayConfig.from_dict(hello.get("config", {}))) + raise _RequestError(f"protocol must be {PROTOCOL_NAME!r}") + if _index(hello.get("protocol_version"), "protocol_version") != PROTOCOL_VERSION: + raise _RequestError(f"protocol_version must be {PROTOCOL_VERSION}") + try: + config = BookReplayConfig.from_dict(hello.get("config", {})) + except (TypeError, ValueError) as exc: + raise _RequestError(str(exc)) from exc + adapter = adapter_factory(config) _write_message( sink, { @@ -72,25 +96,24 @@ def serve_book_replay_stdio( for line in source: if not line.strip(): continue - message = json.loads(line) - if not isinstance(message, dict): - raise ValueError("protocol message must be an object") + message = _message(line) message_type = message.get("type") if message_type == "event": - index = message.get("index") - if isinstance(index, bool) or not isinstance(index, int) or index <= 0: - raise ValueError("event index must be a positive integer") + index = _index(message.get("index"), "event index") if index != last_index + 1: - raise ValueError("event indexes must be contiguous and start at 1") - event = BookReplayEvent.from_mapping(message.get("event", {})) + raise _RequestError("event indexes must be contiguous and start at 1") + try: + event = BookReplayEvent.from_mapping(message.get("event", {})) + except (TypeError, ValueError) as exc: + raise _RequestError(str(exc)) from exc observation = adapter.apply(event, index) if observation.index != index: raise ValueError("adapter returned the wrong observation index") last_index = index _write_message(sink, observation.to_dict(include_type=True)) elif message_type == "snapshot": - if message.get("index") != last_index: - raise ValueError("snapshot index does not match the last event") + if _index(message.get("index"), "snapshot index") != last_index: + raise _RequestError("snapshot index does not match the last event") _write_message( sink, { @@ -100,14 +123,18 @@ def serve_book_replay_stdio( }, ) elif message_type == "finish": - if message.get("event_count") != last_index: - raise ValueError("finish event_count does not match the last event") + if _index(message.get("event_count"), "finish event_count") != last_index: + raise _RequestError("finish event_count does not match the last event") + # Completion includes shutdown. Detach first so a failing close + # produces one error and is never retried by the cleanup path. + closing, adapter = adapter, None + closing.close() _write_message(sink, {"type": "complete", "event_count": last_index}) return 0 else: - raise ValueError(f"unsupported protocol message type: {message_type!r}") - raise ValueError("protocol ended before finish") - except (BrokenPipeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise _RequestError(f"unsupported protocol message type: {message_type!r}") + raise _RequestError("protocol ended before finish") + except (_RequestError, BrokenPipeError) as exc: try: _write_message(sink, {"type": "error", "code": "PROTOCOL_ERROR", "message": str(exc)}) except BrokenPipeError: diff --git a/src/tracebook/book_replay/reference.py b/src/tracebook/book_replay/reference.py index 907b966..fe1c54f 100644 --- a/src/tracebook/book_replay/reference.py +++ b/src/tracebook/book_replay/reference.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass -from decimal import Decimal +from decimal import Decimal, localcontext from typing import Dict, List, Tuple, cast from .._version import __version__ @@ -159,7 +159,15 @@ def _probe(self, event: BookReplayEvent) -> Tuple[SimulatedFill, ...]: quantity=canonical_decimal(quantity), ) ) - remaining -= quantity + # Align both coefficients before subtracting. The ambient + # Decimal context must not erase small FIFO fills. + with localcontext() as context: + exponent = min( + cast(int, remaining.as_tuple().exponent), + cast(int, quantity.as_tuple().exponent), + ) + context.prec = max(remaining.adjusted(), quantity.adjusted()) - exponent + 2 + remaining -= quantity if remaining == 0: return tuple(fills) return tuple(fills) diff --git a/src/tracebook/conformance/model.py b/src/tracebook/conformance/model.py index 9890bc8..0fd86f5 100644 --- a/src/tracebook/conformance/model.py +++ b/src/tracebook/conformance/model.py @@ -54,7 +54,10 @@ def canonical_decimal(value: Any, decimal_places: Optional[int] = None) -> str: raise ConformanceError(f"numeric value cannot be normalized: {value!r}") from exc if number == 0: return "0" - return format(number.normalize(), "f") + # Decimal.normalize() applies the ambient precision and can silently round + # an otherwise exact value. Strip insignificant zeros from fixed-point text. + rendered = format(number, "f") + return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered def _positive_int(value: Any, field_name: str) -> int: diff --git a/tests/fixtures/faulty_book_replay_adapter.py b/tests/fixtures/faulty_book_replay_adapter.py index 4adc250..8525798 100644 --- a/tests/fixtures/faulty_book_replay_adapter.py +++ b/tests/fixtures/faulty_book_replay_adapter.py @@ -12,11 +12,11 @@ class FaultyBookReplayAdapter: - def __init__(self, config): + def __init__(self, config, engine_name="faulty-book-replay-test-adapter"): self._inner = ReferenceBookReplayAdapter(config) self._reorder = False self.metadata = EngineMetadata( - "faulty-book-replay-test-adapter", + engine_name, "1", "Python", ) diff --git a/tests/test_book_replay.py b/tests/test_book_replay.py index fbcf208..2d9528a 100644 --- a/tests/test_book_replay.py +++ b/tests/test_book_replay.py @@ -1,6 +1,7 @@ import io import json import sys +from decimal import Decimal, localcontext from pathlib import Path import pytest @@ -23,12 +24,164 @@ serve_book_replay_stdio, ) from tracebook.book_replay.cli import main +from tracebook.book_replay.model import Outcome, SimulatedFill ROOT = Path(__file__).parents[1] PROFILE_TRACE = ROOT / "src" / "tracebook" / "book_replay" / "fixtures" / f"{PROFILE_NAME}.jsonl" EXAMPLE_ADAPTER = ROOT / "examples" / "book_replay_adapter.py" +def _serve_test(factory, *messages, hello_overrides=None): + hello = dict(type="hello", protocol=PROTOCOL_NAME, protocol_version=PROTOCOL_VERSION) + hello.update(hello_overrides or {}) + source = io.StringIO("".join(json.dumps(item) + "\n" for item in (hello, *messages))) + sink = io.StringIO() + status = serve_book_replay_stdio(factory, source, sink) + return status, [json.loads(line) for line in sink.getvalue().splitlines()] + + +def test_server_close_failure_is_one_terminal_adapter_error(): + instances = [] + + class BrokenClose(ReferenceBookReplayAdapter): + def __init__(self, config): + super().__init__(config) + self.close_calls = 0 + instances.append(self) + + def close(self): + self.close_calls += 1 + raise ValueError("shutdown failed") + + status, messages = _serve_test(BrokenClose, dict(type="finish", event_count=0)) + assert status == 2 + assert [item["type"] for item in messages] == ["ready", "error"] + assert messages[-1]["code"] == "ADAPTER_ERROR" + assert instances[0].close_calls == 1 + + +@pytest.mark.parametrize("stage", ["factory", "apply", "snapshot"]) +@pytest.mark.parametrize("exception", [TypeError, ValueError]) +def test_server_adapter_exceptions_are_not_client_errors(stage, exception): + closed = [] + + class BrokenAdapter(ReferenceBookReplayAdapter): + def __init__(self, config): + if stage == "factory": + raise exception("adapter bug") + super().__init__(config) + + def apply(self, event, index): + if stage == "apply": + raise exception("adapter bug") + return super().apply(event, index) + + def snapshot(self): + raise exception("adapter bug") + + def close(self): + closed.append(True) + + request = ( + dict(type="event", index=1, event=_event().to_dict()) + if stage == "apply" + else dict(type="snapshot", index=0) + ) + status, messages = _serve_test(BrokenAdapter, request) + assert status == 2 + assert messages[-1]["code"] == "ADAPTER_ERROR" + assert len(closed) == (0 if stage == "factory" else 1) + + +@pytest.mark.parametrize("kind,field", [("snapshot", "index"), ("finish", "event_count")]) +@pytest.mark.parametrize("value", [None, False, True, 0.0, "0", -1, "missing"]) +def test_server_requires_integer_session_counters(kind, field, value): + request = {"type": kind} + if value != "missing": + request[field] = value + status, messages = _serve_test(ReferenceBookReplayAdapter, request) + assert status == 2 + assert [item["type"] for item in messages] == ["ready", "error"] + assert messages[-1]["code"] == "PROTOCOL_ERROR" + + +def test_server_invalid_payload_is_a_client_error(): + status, messages = _serve_test( + ReferenceBookReplayAdapter, dict(type="event", index=1, event=None) + ) + assert status == 2 + assert messages[-1]["code"] == "PROTOCOL_ERROR" + status, messages = _serve_test(ReferenceBookReplayAdapter, hello_overrides={"config": None}) + assert status == 2 + assert messages[0]["code"] == "PROTOCOL_ERROR" + + +@pytest.mark.parametrize("field,value", [("outcome", {}), ("fills", (object(),)), ("fills", None)]) +def test_invalid_in_process_observation_is_classified_as_protocol_failure(field, value): + class Malformed(ReferenceBookReplayAdapter): + def apply(self, event, index): + fields = dict( + index=index, + outcome=Outcome("applied"), + fills=(), + state_hash=self.snapshot().digest(), + order_count=0, + ) + fields[field] = value + return BookReplayObservation(**fields) + + report = run_book_replay([_event()], Malformed) + assert report.operational_failure + assert report.divergence.category == "protocol" + assert report.divergence.kind == "adapter_error" + + +@pytest.mark.parametrize("precision", [3, 28, 60]) +def test_probe_subtraction_and_canonical_values_are_exact_at_any_context_precision(precision): + tiny = "0.0000000000000000000000000000000000000001" + leftover = "0." + "9" * 40 + with localcontext() as context: + context.prec = precision + adapter = ReferenceBookReplayAdapter(BookReplayConfig()) + adapter.apply(_event(order_id=1, quantity=tiny), 1) + adapter.apply(_event(order_id=2, quantity="1"), 2) + before = adapter.snapshot() + observation = adapter.apply(_event("probe", quantity="1"), 3) + assert observation.fills == (SimulatedFill("100", tiny), SimulatedFill("100", leftover)) + assert adapter.snapshot() == before + with localcontext() as context: + context.prec = 100 + assert sum(Decimal(fill.quantity) for fill in observation.fills) == Decimal(1) + + +def test_external_timeout_overflow_is_a_public_validation_error(): + with pytest.raises(BookReplayError, match="positive finite"): + ExternalBookReplayAdapter(["never-started"], BookReplayConfig(), timeout_seconds=10**400) + + +def test_external_invalid_metadata_is_a_protocol_error_and_child_is_reaped(monkeypatch): + import subprocess + + processes = [] + popen = subprocess.Popen + + def capture_process(*args, **kwargs): + process = popen(*args, **kwargs) + processes.append(process) + return process + + monkeypatch.setattr(subprocess, "Popen", capture_process) + script = ( + "import json,sys; hello=json.loads(sys.stdin.readline()); " + "hello.update(type='ready',engine={}); print(json.dumps(hello),flush=True); " + "sys.stdin.read()" + ) + with pytest.raises(BookReplayProtocolError, match="invalid ready engine metadata"): + ExternalBookReplayAdapter([sys.executable, "-c", script], BookReplayConfig()) + assert len(processes) == 1 + assert processes[0].poll() is not None + + def _event(op="add", **overrides): values = { "op": op, diff --git a/tests/test_book_replay_artifacts.py b/tests/test_book_replay_artifacts.py index c229d66..2bb9e66 100644 --- a/tests/test_book_replay_artifacts.py +++ b/tests/test_book_replay_artifacts.py @@ -204,6 +204,23 @@ def fail_second(source, target, **kwargs): assert set(tmp_path.iterdir()) == {trace} +@pytest.mark.parametrize("command", ["run", "sample"]) +def test_cli_symlink_loop_is_controlled_before_candidate(tmp_path, monkeypatch, capsys, command): + trace = tmp_path / "trace.jsonl" + _trace(trace) + loop = tmp_path / "loop" + loop.symlink_to(loop) + _forbid_candidate(monkeypatch) + arguments = ( + ["sample", str(loop)] + if command == "sample" + else _command("run", trace, loop / "report.json", tmp_path / "unused") + ) + assert cli.main(arguments) == 2 + assert "Traceback" not in capsys.readouterr().err + assert set(tmp_path.iterdir()) == {trace, loop} + + def test_late_destination_creation_is_never_overwritten(tmp_path, monkeypatch): trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json" _trace(trace) @@ -314,10 +331,65 @@ def interrupt_after_link(source, target, **kwargs): assert set(tmp_path.iterdir()) == {trace} +@pytest.mark.skipif(os.geteuid() == 0, reason="root can deliberately bypass directory permissions") +def test_stage_entry_cannot_be_replaced_between_validation_and_link(tmp_path, monkeypatch): + report = tmp_path / "report.json" + link = os.link + + def replace_stage(source, target, **kwargs): + # The old implementation allowed this unlink, then linked the foreign + # replacement and left it at report.json after its failed validation. + os.unlink(source, dir_fd=kwargs["src_dir_fd"]) + descriptor = os.open( + source, os.O_WRONLY | os.O_CREAT | os.O_EXCL, dir_fd=kwargs["src_dir_fd"] + ) + os.write(descriptor, b"foreign bytes") + os.close(descriptor) + link(source, target, **kwargs) + + monkeypatch.setattr(artifacts.os, "link", replace_stage) + monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {replace_stage}) + with artifacts.reserve_outputs(str(report)) as outputs: + with pytest.raises(PermissionError): + artifacts.publish_outputs([(outputs[str(report)], b"owned bytes")]) + assert not report.exists() + assert list(tmp_path.iterdir()) == [] + + +def test_replacing_stage_directory_path_cannot_change_published_inode(tmp_path, monkeypatch): + report = tmp_path / "report.json" + link = os.link + with artifacts.reserve_outputs(str(report)) as outputs: + output = outputs[str(report)] + stage = tmp_path / output._stage_directory_name + moved = tmp_path / "moved-stage" + + def replace_directory(source, target, **kwargs): + # macOS requires write permission on a directory being renamed; + # emulate a writer that can rename it, then restore its protection. + stage.chmod(0o700) + stage.rename(moved) + moved.chmod(0o500) + stage.mkdir() + (stage / source).write_bytes(b"foreign replacement") + link(source, target, **kwargs) + + monkeypatch.setattr(artifacts.os, "link", replace_directory) + artifacts.publish_outputs([(output, b"owned bytes")]) + assert report.read_bytes() == b"owned bytes" + assert (stage / output._stage_name).read_bytes() == b"foreign replacement" + assert list(moved.iterdir()) == [] + + def test_cleanup_failure_still_closes_all_descriptors_and_other_sidecars(tmp_path, monkeypatch): reservation = artifacts.OutputReservation(tmp_path / "report.json") reservation.__enter__() - descriptors = (reservation._stage_fd, reservation._lock_fd, reservation._parent_fd) + descriptors = ( + reservation._stage_fd, + reservation._lock_fd, + reservation._stage_directory_fd, + reservation._parent_fd, + ) unlink = os.unlink def fail_stage_cleanup(name, **kwargs): @@ -332,7 +404,8 @@ def fail_stage_cleanup(name, **kwargs): for descriptor in descriptors: with pytest.raises(OSError): os.fstat(descriptor) - assert {path.name for path in tmp_path.iterdir()} == {reservation._stage_name} + assert {path.name for path in tmp_path.iterdir()} == {reservation._stage_directory_name} + assert (tmp_path / reservation._stage_directory_name / reservation._stage_name).is_file() def test_trace_serialization_failure_precedes_any_artifact_write(tmp_path, monkeypatch): diff --git a/tests/test_book_replay_campaign.py b/tests/test_book_replay_campaign.py index d0f4c54..82a720f 100644 --- a/tests/test_book_replay_campaign.py +++ b/tests/test_book_replay_campaign.py @@ -3,15 +3,12 @@ from pathlib import Path import pytest +from tests.fixtures.faulty_book_replay_adapter import FaultyBookReplayAdapter from tracebook.book_replay import ( BOOK_REPLAY_CAPABILITIES, BookReplayError, BookReplayEvent, - BookReplayObservation, - BookReplaySnapshot, - BookReplayState, - EngineMetadata, ReferenceBookReplayAdapter, generate_book_replay_trace, measure_book_replay_coverage, @@ -25,39 +22,17 @@ FAULTY_ADAPTER = ROOT / "tests" / "fixtures" / "faulty_book_replay_adapter.py" -class _ReorderingAdapter: +class _ReorderingAdapter(FaultyBookReplayAdapter): def __init__(self, config): - self._inner = ReferenceBookReplayAdapter(config) - self._reorder = False - self.metadata = EngineMetadata("reordering-test-adapter", "1", "Python") - - def apply(self, event, index): - observation = self._inner.apply(event, index) - if event.op == "update" and event.order_id == 1: - self._reorder = True - state = self.snapshot() - return BookReplayObservation( - index, - observation.outcome, - observation.fills, - state.digest(), - state.order_count, - ) - - def snapshot(self): - state = self._inner.snapshot() - if not self._reorder: - return state - books = [] - for book in state.books: - bids = list(book.bids) - if book.symbol == "TEST" and len(bids) >= 2: - bids[0], bids[1] = bids[1], bids[0] - books.append(BookReplaySnapshot(book.symbol, tuple(bids), book.asks)) - return BookReplayState(tuple(books)) - - def close(self): - self._inner.close() + super().__init__(config, engine_name="reordering-test-adapter") + + +def test_side_only_update_does_not_claim_price_relocation_coverage(): + add = BookReplayEvent("add", "TEST", 1, "BUY", "100", "1") + side_change = BookReplayEvent("update", "TEST", 1, "SELL", "100", "1") + price_change = BookReplayEvent("update", "TEST", 1, "SELL", "101", "1") + assert "price-relocation" not in measure_book_replay_coverage([[add, side_change]]).covered + assert "price-relocation" in measure_book_replay_coverage([[add, price_change]]).covered def test_generator_is_deterministic_and_scaffold_covers_every_capability(): diff --git a/tests/test_distribution_privacy.py b/tests/test_distribution_privacy.py index 509578e..c8bbec9 100644 --- a/tests/test_distribution_privacy.py +++ b/tests/test_distribution_privacy.py @@ -8,6 +8,7 @@ import sys import tarfile import zipfile +import zlib from pathlib import Path import pytest @@ -15,6 +16,24 @@ from tools.verify_distribution_privacy import PrivacyError, main, verify_artifact ROOT = Path(__file__).resolve().parents[1] + + +def test_privacy_gate_handles_corrupt_deflated_inventory(tmp_path, capsys): + artifact = tmp_path / "corrupt.whl" + with zipfile.ZipFile(artifact, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("fixture.egg-info/SOURCES.txt", "pyproject.toml\n") + # Preserve the ZIP index, but corrupt the compressed block's type bits. + data = bytearray(artifact.read_bytes()) + filename_length = int.from_bytes(data[26:28], "little") + extra_length = int.from_bytes(data[28:30], "little") + data[30 + filename_length + extra_length] = 7 + artifact.write_bytes(data) + with pytest.raises(zlib.error): + verify_artifact(artifact) + assert main([str(artifact)]) == 1 + assert "distribution privacy check failed" in capsys.readouterr().err + + PRIVATE_PATHS = ( "experiments/private/agent-qualification-v2/gold.json", "experiments/private/agent-qualification-v2/execution/controller.py", diff --git a/tests/test_integration_workflow_coverage.py b/tests/test_integration_workflow_coverage.py index f6d2971..eccf8ec 100644 --- a/tests/test_integration_workflow_coverage.py +++ b/tests/test_integration_workflow_coverage.py @@ -17,18 +17,79 @@ } -def _pull_request_paths(workflow): - source = (ROOT / ".github" / "workflows" / f"{workflow}.yml").read_text(encoding="utf-8") - # Read this deliberately simple YAML sequence without adding PyYAML to the - # dependency-free package's test requirements. Do not search job bodies, - # where a mentioned path cannot cause the workflow to run. - match = re.search(r"^ pull_request:\n paths:\n((?: - .+\n)+)", source, re.M) - assert match is not None, f"{workflow} must declare pull_request.paths" - patterns = [ast.literal_eval(line.removeprefix(" - ")) for line in match[1].splitlines()] +def _mapping_block(lines, key): + """Read a block in our workflows' block-style YAML subset, not job text.""" + lines = [line for line in lines if line.strip() and not line.lstrip().startswith("#")] + assert lines, f"missing YAML mapping: {key}" + base = min(len(line) - len(line.lstrip()) for line in lines) + for index, line in enumerate(lines): + indent = len(line) - len(line.lstrip()) + if indent != base or not re.fullmatch(rf"['\"]?{key}['\"]?:\s*(?:#.*)?", line.strip()): + continue + block = [] + for child in lines[index + 1 :]: + if len(child) - len(child.lstrip()) <= indent: + break + block.append(child) + return block + raise AssertionError(f"missing YAML mapping: {key}") + + +def _parse_pull_request_paths(source): + # Intentionally support block mappings and quoted positive path sequences + # used here. Unsupported YAML fails closed, without a new runtime dependency. + block = source.splitlines() + for key in ("on", "pull_request", "paths"): + block = _mapping_block(block, key) + assert block and all(line.strip().startswith("- ") for line in block) + patterns = [ast.literal_eval(line.strip().removeprefix("- ")) for line in block] assert all(isinstance(pattern, str) and not pattern.startswith("!") for pattern in patterns) return patterns +def _pull_request_paths(workflow): + source = (ROOT / ".github" / "workflows" / f"{workflow}.yml").read_text(encoding="utf-8") + return _parse_pull_request_paths(source) + + +@pytest.mark.parametrize("indent", [2, 4]) +def test_path_reader_allows_comments_blank_lines_and_other_trigger_keys(indent): + space = " " * indent + source = "\n".join( + [ + "on:", + f"{space}pull_request: # keep native proofs attached", + f"{space * 2}branches:", + f"{space * 3}- 'main'", + "", + f"{space * 2}# runtime inputs", + f"{space * 2}paths:", + f"{space * 3}- 'src/**' # reference", + "", + f"{space * 3}# shared crate", + f'{space * 3}- "integrations/**"', + f"{space}push:", + f"{space * 2}paths:", + f"{space * 3}- 'ignored/**'", + "jobs:", + f"{space}pull_request:", + f"{space * 2}paths:", + f"{space * 3}- 'wrong/**'", + ] + ) + assert _parse_pull_request_paths(source) == ["src/**", "integrations/**"] + + +def test_path_reader_cannot_use_paths_from_jobs_or_another_trigger(): + for source in ( + "jobs:\n pull_request:\n paths:\n - 'src/**'", + "on:\n push:\n paths:\n - 'src/**'\n" + " pull_request:\n branches:\n - 'main'", + ): + with pytest.raises(AssertionError, match="missing YAML mapping"): + _parse_pull_request_paths(source) + + def _matches(path, pattern): # The checked workflows use only literal paths and positive * / ** globs. # Unlike fnmatch, a single * must not match a directory separator. diff --git a/tools/verify_distribution_privacy.py b/tools/verify_distribution_privacy.py index 79694dd..f67fadf 100644 --- a/tools/verify_distribution_privacy.py +++ b/tools/verify_distribution_privacy.py @@ -12,6 +12,7 @@ import sys import tarfile import zipfile +import zlib from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Sequence @@ -112,7 +113,7 @@ def main(argv: Sequence[str] | None = None) -> int: try: for artifact in args.artifacts: verify_artifact(artifact) - except (PrivacyError, OSError, tarfile.TarError, zipfile.BadZipFile) as error: + except (PrivacyError, OSError, tarfile.TarError, zipfile.BadZipFile, zlib.error) as error: print(f"distribution privacy check failed: {error}", file=sys.stderr) return 1 print(f"Distribution privacy verified for {len(args.artifacts)} artifacts.") From f035e0c162ce52c808c3216d75ebb21911194c48 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Sat, 5 Sep 2026 02:49:37 -0400 Subject: [PATCH 17/19] Fix remaining release-code review follow-ups --- CHANGELOG.md | 3 ++ docs/pr-84-review-disposition.md | 32 ++++++++++++- integrations/rust_protocol/src/quantity.rs | 17 ++++++- src/tracebook/book_replay/artifacts.py | 46 +++++++++++++++---- src/tracebook/book_replay/cli.py | 10 ++-- src/tracebook/book_replay/protocol.py | 2 +- tests/test_book_replay.py | 24 ++++++++-- tests/test_book_replay_artifacts.py | 53 ++++++++++++++++++++++ 8 files changed, 168 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63a9d23..cdbceda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions. ## Unreleased +- Fixed four PR #84 follow-ups: clean owned staging directories when opening + fails, classify adapter broken-pipe failures correctly, distinguish zero + quantities from precision loss, and handle symlink loops in CLI input paths. - Addressed the release-code findings from PR #84: fail closed on L3 shutdown and malformed protocol values, preserve exact decimal probe arithmetic, protect staged artifacts against ordinary source-path replacement, reject diff --git a/docs/pr-84-review-disposition.md b/docs/pr-84-review-disposition.md index 0547b6a..e0a95f4 100644 --- a/docs/pr-84-review-disposition.md +++ b/docs/pr-84-review-disposition.md @@ -1,7 +1,8 @@ # PR #84 Review Disposition This records the 30 comments in the initial automated review of -[PR #84](https://github.com/Taz33m/tracebook/pull/84), against `0c98e18`. +[PR #84](https://github.com/Taz33m/tracebook/pull/84), against `0c98e18`, +and the four follow-up comments against `e65416a`. It is a disposition of that review, not release approval or a new experiment result. Source and regression tests, not the review's severity labels, determine the implemented changes. @@ -45,6 +46,25 @@ Two limits matter when interpreting these fixes: not overwrite destinations, and rollback preserves competing files. The directory-permission test is intentionally skipped when running as root. +## Follow-up release-code fixes + +All four follow-up findings are addressed without changing positive-value +normalization, matching semantics, or frozen research files. + +| Comment ID | Change | Regression / verification surface | +| --- | --- | --- | +| 3939212113 | Retain the staging directory's identity before opening it, remove only a matching entry on setup failure, and reject an opened replacement before writing or changing its permissions. | `test_stage_directory_setup_failure_cleans_only_owned_entries`: failed open, competing replacement, and descriptor closure | +| 3939212117 | Reserve `PROTOCOL_ERROR` for explicit client validation failures; adapter `BrokenPipeError` follows the adapter-error path, including shutdown. | Close/factory/apply/snapshot regressions plus a broken output pipe; close remains single-shot | +| 3939212124 | Diagnose an exact zero as a nonpositive quantity; retain the separate precision-loss error for positive quantities that round to zero. | Shared Rust zero-diagnostic test at 0, 12, and 18 places, plus existing positive rounding tests | +| 3939212131 | Use the same narrow path-resolution guard for CLI inputs and distinct-path checks. | Actual input symlink loops for both `run` and `minimize`, including Python 3.10's `RuntimeError` behavior | + +The still-open duplicate-fixture comment, `3939089005`, is already addressed +by `e65416a`: `tests/test_book_replay_campaign.py` imports the shared +`FaultyBookReplayAdapter`. Its small `_ReorderingAdapter` subclass only supplies +the historical engine name; it contains no fault, snapshot, or shutdown logic. +Existing campaign and reduced-failure identities remain covered by regression +tests. This is a stale release-code conversation, not a deferred research issue. + ## Frozen research findings — open, not silently patched These 13 findings are in the historical/hash-bound research harness. The @@ -78,7 +98,7 @@ belong in a provider request or this public document. ## Release gate -Local verification of the fixes passed: Python 3.13 had 651 passing tests, +Local verification of the initial fixes passed: Python 3.13 had 651 passing tests, one skip, and 82.38% coverage; the focused Python 3.10 regressions passed too. Formatting, lint, type checking, static security checks, pinned Rust 1.88/Go 1.23.5 checks, distribution privacy, and sdist/wheel agreement passed. @@ -88,6 +108,14 @@ the Nautilus 2,500-event positive campaign/three-event negative control retained their expected identities and reduced trace. These are local checks; consult the PR for the latest remote CI and review state. +Follow-up verification passed separately: Python 3.13 had 661 passing tests, +one skip, and 82.38% coverage. The focused L3 protocol, artifact, and campaign +suite passed all 114 tests on Python 3.10. All 33 Rust tests across the shared +protocol and both adapters passed with Rust 1.88, along with clippy and rustfmt. +Python formatting, lint, type checking, static security checks, and compilation +also passed. This follow-up did not recollect provider runs or rewrite retained +qualification artifacts; the initial verification above remains historical. + Keep the research conversations open until their disposition is agreed. Passing release tests does not justify dismissing those comments, bypassing repository protections, merging the PR, or publishing a tag. The previous maintainer diff --git a/integrations/rust_protocol/src/quantity.rs b/integrations/rust_protocol/src/quantity.rs index 41009b8..25ab00d 100644 --- a/integrations/rust_protocol/src/quantity.rs +++ b/integrations/rust_protocol/src/quantity.rs @@ -61,6 +61,9 @@ impl QuantityEncoding { } pub fn format(&self, units: u64) -> Result { + if units == 0 { + return Err("quantity must be positive".to_string()); + } let places = self.output_decimal_places.min(NATIVE_DECIMAL_PLACES); let divisor = 10_u64.pow(NATIVE_DECIMAL_PLACES - places); let mut rounded = units / divisor; @@ -200,6 +203,18 @@ mod tests { .contains("cannot be represented") ); } - assert!(QuantityEncoding::new(12).unwrap().format(0).is_err()); + } + + #[test] + fn zero_observations_report_nonpositive_quantity_not_precision_loss() { + for places in [0, 12, 18] { + assert_eq!( + QuantityEncoding::new(places) + .unwrap() + .format(0) + .unwrap_err(), + "quantity must be positive" + ); + } } } diff --git a/src/tracebook/book_replay/artifacts.py b/src/tracebook/book_replay/artifacts.py index 12f2eea..9fdc5c7 100644 --- a/src/tracebook/book_replay/artifacts.py +++ b/src/tracebook/book_replay/artifacts.py @@ -37,6 +37,7 @@ def __init__(self, destination: str | Path) -> None: self._stage_name = "payload" self._parent_fd: Optional[int] = None self._stage_directory_fd: Optional[int] = None + self._stage_directory_identity: Optional[tuple[int, int]] = None self._lock_fd: Optional[int] = None self._stage_fd: Optional[int] = None self._published = False @@ -56,11 +57,23 @@ def __enter__(self) -> "OutputReservation": flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) self._lock_fd = os.open(self._lock_name, flags, 0o600, dir_fd=self._parent_fd) os.mkdir(self._stage_directory_name, mode=0o700, dir_fd=self._parent_fd) - self._stage_directory_fd = os.open( + # Retain the identity even if opening the newly created directory + # fails. Cleanup must not remove a replacement at the same name. + self._stage_directory_identity = _identity( + os.stat(self._stage_directory_name, dir_fd=self._parent_fd, follow_symlinks=False) + ) + stage_directory_fd = os.open( self._stage_directory_name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=self._parent_fd, ) + try: + if _identity(os.fstat(stage_directory_fd)) != self._stage_directory_identity: + raise BookReplayError(f"book-replay output reservation changed: {self.target}") + except BaseException: + os.close(stage_directory_fd) + raise + self._stage_directory_fd = stage_directory_fd self._stage_fd = os.open( self._stage_name, flags, 0o400, dir_fd=self._stage_directory_fd ) @@ -120,6 +133,21 @@ def validate(self) -> None: raise BookReplayError(f"book-replay output reservation changed: {self.target}") self._require_absent() + def _owns_stage_directory(self) -> bool: + if self._parent_fd is None or self._stage_directory_identity is None: + return False + try: + return ( + _identity( + os.stat( + self._stage_directory_name, dir_fd=self._parent_fd, follow_symlinks=False + ) + ) + == self._stage_directory_identity + ) + except OSError: + return False + def _same_parent(self) -> bool: if self._parent_fd is None: return False @@ -190,18 +218,18 @@ def close(self) -> None: except OSError as exc: error = error or exc self._stage_fd = self._lock_fd = None + try: + if self._owns_stage_directory(): + os.rmdir(self._stage_directory_name, dir_fd=self._parent_fd) + except OSError as exc: + error = error or exc + self._stage_directory_identity = None if self._stage_directory_fd is not None: try: - if self._owns_entry(self._stage_directory_name, self._stage_directory_fd): - os.rmdir(self._stage_directory_name, dir_fd=self._parent_fd) + os.close(self._stage_directory_fd) except OSError as exc: error = error or exc - finally: - try: - os.close(self._stage_directory_fd) - except OSError as exc: - error = error or exc - self._stage_directory_fd = None + self._stage_directory_fd = None if self._parent_fd is not None: try: os.close(self._parent_fd) diff --git a/src/tracebook/book_replay/cli.py b/src/tracebook/book_replay/cli.py index 60ae764..127d609 100644 --- a/src/tracebook/book_replay/cli.py +++ b/src/tracebook/book_replay/cli.py @@ -124,11 +124,15 @@ def _emit( print(f"Report written: {Path(output)}") -def _require_distinct_paths(*paths: Optional[str]) -> None: +def _resolve_path(path: str) -> Path: try: - resolved = [Path(path).expanduser().resolve() for path in paths if path is not None] + return Path(path).expanduser().resolve() except RuntimeError as exc: raise BookReplayError(f"cannot resolve book-replay path: {exc}") from exc + + +def _require_distinct_paths(*paths: Optional[str]) -> None: + resolved = [_resolve_path(path) for path in paths if path is not None] if len(resolved) != len(set(resolved)) or any( left in right.parents or right in left.parents for index, left in enumerate(resolved) @@ -171,7 +175,7 @@ def main(argv: Optional[List[str]] = None) -> int: return 0 return 2 if campaign.failure.minimization.report.operational_failure else 1 - events_path = Path(args.events).expanduser().resolve() + events_path = _resolve_path(args.events) events_output = args.events_output if args.command == "minimize" else None _require_distinct_paths(str(events_path), args.output, events_output) with reserve_outputs(args.output, events_output) as outputs: diff --git a/src/tracebook/book_replay/protocol.py b/src/tracebook/book_replay/protocol.py index 2fa16e4..efba6b7 100644 --- a/src/tracebook/book_replay/protocol.py +++ b/src/tracebook/book_replay/protocol.py @@ -134,7 +134,7 @@ def serve_book_replay_stdio( else: raise _RequestError(f"unsupported protocol message type: {message_type!r}") raise _RequestError("protocol ended before finish") - except (_RequestError, BrokenPipeError) as exc: + except _RequestError as exc: try: _write_message(sink, {"type": "error", "code": "PROTOCOL_ERROR", "message": str(exc)}) except BrokenPipeError: diff --git a/tests/test_book_replay.py b/tests/test_book_replay.py index 2d9528a..065787b 100644 --- a/tests/test_book_replay.py +++ b/tests/test_book_replay.py @@ -40,7 +40,8 @@ def _serve_test(factory, *messages, hello_overrides=None): return status, [json.loads(line) for line in sink.getvalue().splitlines()] -def test_server_close_failure_is_one_terminal_adapter_error(): +@pytest.mark.parametrize("exception", [ValueError, BrokenPipeError]) +def test_server_close_failure_is_one_terminal_adapter_error(exception): instances = [] class BrokenClose(ReferenceBookReplayAdapter): @@ -51,7 +52,7 @@ def __init__(self, config): def close(self): self.close_calls += 1 - raise ValueError("shutdown failed") + raise exception("shutdown failed") status, messages = _serve_test(BrokenClose, dict(type="finish", event_count=0)) assert status == 2 @@ -61,7 +62,7 @@ def close(self): @pytest.mark.parametrize("stage", ["factory", "apply", "snapshot"]) -@pytest.mark.parametrize("exception", [TypeError, ValueError]) +@pytest.mark.parametrize("exception", [TypeError, ValueError, BrokenPipeError]) def test_server_adapter_exceptions_are_not_client_errors(stage, exception): closed = [] @@ -93,6 +94,23 @@ def close(self): assert len(closed) == (0 if stage == "factory" else 1) +def test_server_broken_output_pipe_returns_failure_and_closes_adapter_once(): + closed = [] + + class Adapter(ReferenceBookReplayAdapter): + def close(self): + closed.append(True) + + class BrokenSink(io.StringIO): + def write(self, value): + raise BrokenPipeError("output pipe closed") + + hello = dict(type="hello", protocol=PROTOCOL_NAME, protocol_version=PROTOCOL_VERSION) + source = io.StringIO(json.dumps(hello) + "\n") + assert serve_book_replay_stdio(Adapter, source, BrokenSink()) == 2 + assert closed == [True] + + @pytest.mark.parametrize("kind,field", [("snapshot", "index"), ("finish", "event_count")]) @pytest.mark.parametrize("value", [None, False, True, 0.0, "0", -1, "missing"]) def test_server_requires_integer_session_counters(kind, field, value): diff --git a/tests/test_book_replay_artifacts.py b/tests/test_book_replay_artifacts.py index 2bb9e66..532872f 100644 --- a/tests/test_book_replay_artifacts.py +++ b/tests/test_book_replay_artifacts.py @@ -221,6 +221,22 @@ def test_cli_symlink_loop_is_controlled_before_candidate(tmp_path, monkeypatch, assert set(tmp_path.iterdir()) == {trace, loop} +@pytest.mark.parametrize("command", ["run", "minimize"]) +def test_cli_input_symlink_loop_is_controlled_before_candidate( + tmp_path, monkeypatch, capsys, command +): + loop = tmp_path / "loop" + loop.symlink_to(loop) + _forbid_candidate(monkeypatch) + + arguments = _command(command, loop, tmp_path / "report.json", tmp_path / "reduced.jsonl") + assert cli.main(arguments) == 2 + error = capsys.readouterr().err + assert error.startswith("Error:") + assert "Traceback" not in error + assert set(tmp_path.iterdir()) == {loop} + + def test_late_destination_creation_is_never_overwritten(tmp_path, monkeypatch): trace, report = tmp_path / "trace.jsonl", tmp_path / "report.json" _trace(trace) @@ -381,6 +397,43 @@ def replace_directory(source, target, **kwargs): assert list(moved.iterdir()) == [] +@pytest.mark.parametrize( + "replace_directory,fail_open", [(False, True), (True, True), (True, False)] +) +def test_stage_directory_setup_failure_cleans_only_owned_entries( + tmp_path, monkeypatch, replace_directory, fail_open +): + reservation = artifacts.OutputReservation(tmp_path / "report.json") + stage = tmp_path / reservation._stage_directory_name + moved = tmp_path / "moved-stage" + open_descriptor = os.open + descriptors = [] + + def inject_stage_open(path, *args, **kwargs): + if path == reservation._stage_directory_name: + if replace_directory: + stage.rename(moved) + stage.mkdir(mode=0o500) + if fail_open: + raise OSError("injected stage directory open failure") + descriptor = open_descriptor(path, *args, **kwargs) + descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(artifacts.os, "open", inject_stage_open) + monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {inject_stage_open}) + error = OSError if fail_open else artifacts.BookReplayError + with pytest.raises(error, match="open failure" if fail_open else "reservation changed"): + reservation.__enter__() + for descriptor in descriptors: + with pytest.raises(OSError): + os.fstat(descriptor) + assert set(tmp_path.iterdir()) == ({stage, moved} if replace_directory else set()) + if replace_directory: + assert list(stage.iterdir()) == list(moved.iterdir()) == [] + assert stage.stat().st_mode & 0o777 == 0o500 + + def test_cleanup_failure_still_closes_all_descriptors_and_other_sidecars(tmp_path, monkeypatch): reservation = artifacts.OutputReservation(tmp_path / "report.json") reservation.__enter__() From 38ef03ff0d64d85d029f045d637fd171e71bc254 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Sat, 5 Sep 2026 04:15:37 -0400 Subject: [PATCH 18/19] Record agreed deferral of frozen research findings to issue 85 --- docs/pr-84-review-disposition.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/pr-84-review-disposition.md b/docs/pr-84-review-disposition.md index e0a95f4..ea94fb3 100644 --- a/docs/pr-84-review-disposition.md +++ b/docs/pr-84-review-disposition.md @@ -58,19 +58,25 @@ normalization, matching semantics, or frozen research files. | 3939212124 | Diagnose an exact zero as a nonpositive quantity; retain the separate precision-loss error for positive quantities that round to zero. | Shared Rust zero-diagnostic test at 0, 12, and 18 places, plus existing positive rounding tests | | 3939212131 | Use the same narrow path-resolution guard for CLI inputs and distinct-path checks. | Actual input symlink loops for both `run` and `minimize`, including Python 3.10's `RuntimeError` behavior | -The still-open duplicate-fixture comment, `3939089005`, is already addressed +The duplicate-fixture comment, `3939089005`, was already addressed by `e65416a`: `tests/test_book_replay_campaign.py` imports the shared `FaultyBookReplayAdapter`. Its small `_ReorderingAdapter` subclass only supplies the historical engine name; it contains no fault, snapshot, or shutdown logic. Existing campaign and reduced-failure identities remain covered by regression -tests. This is a stale release-code conversation, not a deferred research issue. +tests. Its stale release-code conversation was resolved with source and test +evidence; it is not a deferred research issue. -## Frozen research findings — open, not silently patched +## Frozen research findings — explicitly deferred, not fixed These 13 findings are in the historical/hash-bound research harness. The accepted scope leaves that code, frozen inputs, evaluator material, and retained -results unchanged. They remain open pending a separate research disposition; -the release fixes above do not resolve them. +results unchanged. On 2026-09-05 the maintainer explicitly authorized transferring +all 13 findings to [follow-up issue #85](https://github.com/Taz33m/tracebook/issues/85) +so this release-code PR can merge after its remaining fix and checks. That issue +preserves every original review link and an unchecked item for each finding. +The PR conversations are resolved only as an agreed transfer: these findings +remain open in the follow-up, not fixed or dismissed as false positives. The +release fixes above do not resolve the underlying research risks. | Comment ID | Reported issue | Required next decision/check | | --- | --- | --- | @@ -116,7 +122,10 @@ Python formatting, lint, type checking, static security checks, and compilation also passed. This follow-up did not recollect provider runs or rewrite retained qualification artifacts; the initial verification above remains historical. -Keep the research conversations open until their disposition is agreed. Passing -release tests does not justify dismissing those comments, bypassing repository -protections, merging the PR, or publishing a tag. The previous maintainer -invitation and its immutable source/wheel references remain unchanged. +The agreed transfer to issue #85 supersedes the earlier instruction to keep +these 13 PR conversations open pending a disposition. Keep the follow-up issue +open until its evidence-backed completion criteria are reviewed. Passing release +tests does not dismiss those risks or justify bypassing repository protections. +The remaining release-code review and required checks must pass before the +authorized merge; no tag or package publication is authorized. The previous +maintainer invitation and its immutable source/wheel references remain unchanged. From 4ba428eadf7af34343b0eba3977c6743a7817c03 Mon Sep 17 00:00:00 2001 From: Tazeem Mahashin Date: Sat, 5 Sep 2026 04:25:26 -0400 Subject: [PATCH 19/19] Report unverified staging residue without unsafe cleanup --- CHANGELOG.md | 3 + docs/book-replay-conformance.md | 16 ++++ docs/pr-84-review-disposition.md | 31 ++++++- src/tracebook/book_replay/artifacts.py | 19 ++++- tests/test_book_replay_artifacts.py | 108 +++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdbceda..d72792e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The project follows [Keep a Changelog](https://keepachangelog.com/) conventions. ## Unreleased +- Made initial staging-identity failures explicit: report the possible residue + path and skipped cleanup, preserve unverified entries, and exit before starting + a candidate. Documented conservative manual recovery without blanket deletion. - Fixed four PR #84 follow-ups: clean owned staging directories when opening fails, classify adapter broken-pipe failures correctly, distinguish zero quantities from precision loss, and handle symlink loops in CLI input paths. diff --git a/docs/book-replay-conformance.md b/docs/book-replay-conformance.md index 59d46bc..00896db 100644 --- a/docs/book-replay-conformance.md +++ b/docs/book-replay-conformance.md @@ -118,6 +118,22 @@ leave sidecars. After a forced stop or cleanup failure, retain the artifacts and use fresh output names for a new invocation; do not remove a live process's reservations. +If the first metadata read after creating a staging directory fails, the +command exits `2` before starting a candidate. It explicitly reports that +ownership could not be verified, automatic directory cleanup was skipped, and +the original path of possible staging residue. That entry is left untouched: +it may be the empty directory from this attempt, a competing replacement, or +already moved/deleted. Owned lock cleanup is still attempted and opened +descriptors are closed. This is a reported cleanup limitation, not a guarantee +that every failed setup leaves no sidecars. + +For recovery, first ensure no process is using the output directory. Inspect +the reported path and any parent-directory moves; the diagnostic is not proof +of ownership. Remove only an empty directory independently confirmed to belong +to the failed attempt, using non-recursive removal. Leave uncertain entries, +symlinks, other writers' data, and nonempty directories untouched, and use fresh +output names. Do not run a blanket cleanup of `.tracebook-stage-*` paths. + Staged payloads live in owner-only, read-only directories and publication uses the open directory descriptor, so replacing a staging directory's parent entry cannot substitute another payload. Published files are owner-readable. This diff --git a/docs/pr-84-review-disposition.md b/docs/pr-84-review-disposition.md index ea94fb3..dd74df3 100644 --- a/docs/pr-84-review-disposition.md +++ b/docs/pr-84-review-disposition.md @@ -2,7 +2,8 @@ This records the 30 comments in the initial automated review of [PR #84](https://github.com/Taz33m/tracebook/pull/84), against `0c98e18`, -and the four follow-up comments against `e65416a`. +the four follow-up comments against `e65416a`, and the cleanup follow-up +against `f035e0c`. It is a disposition of that review, not release approval or a new experiment result. Source and regression tests, not the review's severity labels, determine the implemented changes. @@ -66,6 +67,28 @@ Existing campaign and reduced-failure identities remain covered by regression tests. Its stale release-code conversation was resolved with source and test evidence; it is not a deferred research issue. +## Initial staging identity failure — agreed conservative handling + +Comment [3939788196](https://github.com/Taz33m/tracebook/pull/84#discussion_r3939788196) +correctly identifies a possible empty-directory residue when the first `stat` +after `mkdir` fails. With neither an identity nor a directory descriptor, the +runner cannot safely distinguish that entry from a competing replacement. + +The maintainer explicitly approved retaining an unverified entry, reporting +an explicit error, and adding tests/documentation before merge. The runner now +raises `BookReplayError` with the original possible-residue path and the reason +automatic directory cleanup was skipped. CLI commands exit `2` before candidate +execution, retain the underlying exception as the cause, and still attempt +owned-lock cleanup and close open descriptors. Known-ownership cleanup remains +unchanged. This addresses silent residue through an agreed, observable failure +contract; it does not claim automatic removal when ownership is unknown. + +Regression tests cover the original empty directory, empty and populated +competing directories, a replacement symlink, a moved-away entry, descriptor +closure, and all four CLI commands. Recovery instructions in +[book-replay conformance](book-replay-conformance.md) require independently +verified ownership and prohibit blanket or recursive removal of uncertain paths. + ## Frozen research findings — explicitly deferred, not fixed These 13 findings are in the historical/hash-bound research harness. The @@ -122,6 +145,12 @@ Python formatting, lint, type checking, static security checks, and compilation also passed. This follow-up did not recollect provider runs or rewrite retained qualification artifacts; the initial verification above remains historical. +The final initial-identity error handling passed 670 Python 3.13 tests with one +skip and 82.39% coverage, plus all 123 focused L3 tests on Python 3.10. Its nine +new regression cases failed before the change and passed afterward. Formatting, +lint, type checking, static security checks, compilation, and whitespace checks +passed. No frozen research files or retained qualification artifacts changed. + The agreed transfer to issue #85 supersedes the earlier instruction to keep these 13 PR conversations open pending a disposition. Keep the follow-up issue open until its evidence-backed completion criteria are reviewed. Passing release diff --git a/src/tracebook/book_replay/artifacts.py b/src/tracebook/book_replay/artifacts.py index 9fdc5c7..376ffec 100644 --- a/src/tracebook/book_replay/artifacts.py +++ b/src/tracebook/book_replay/artifacts.py @@ -59,9 +59,22 @@ def __enter__(self) -> "OutputReservation": os.mkdir(self._stage_directory_name, mode=0o700, dir_fd=self._parent_fd) # Retain the identity even if opening the newly created directory # fails. Cleanup must not remove a replacement at the same name. - self._stage_directory_identity = _identity( - os.stat(self._stage_directory_name, dir_fd=self._parent_fd, follow_symlinks=False) - ) + try: + self._stage_directory_identity = _identity( + os.stat( + self._stage_directory_name, dir_fd=self._parent_fd, follow_symlinks=False + ) + ) + except OSError as exc: + # mkdir gives no inode/descriptor. Do not guess ownership or + # remove a replacement when this first identity read fails. + residue = self.target.parent / self._stage_directory_name + raise BookReplayError( + "staging directory ownership could not be verified; " + "automatic directory cleanup skipped; " + f"possible staging residue at original path {str(residue)!r}. " + f"Inspect its location and ownership before manual cleanup: {exc}" + ) from exc stage_directory_fd = os.open( self._stage_directory_name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), diff --git a/tests/test_book_replay_artifacts.py b/tests/test_book_replay_artifacts.py index 532872f..1461a8c 100644 --- a/tests/test_book_replay_artifacts.py +++ b/tests/test_book_replay_artifacts.py @@ -434,6 +434,114 @@ def inject_stage_open(path, *args, **kwargs): assert stage.stat().st_mode & 0o777 == 0o500 +@pytest.mark.parametrize("replacement", ["none", "empty", "populated", "symlink", "missing"]) +def test_initial_stage_stat_failure_reports_unverified_entry_without_removing_it( + tmp_path, monkeypatch, replacement +): + reservation = artifacts.OutputReservation(tmp_path / "report.json") + stage = tmp_path / reservation._stage_directory_name + moved = tmp_path / "moved-stage" + stat, open_descriptor = os.stat, os.open + descriptors = [] + failure = OSError("injected initial staging stat failure") + + def fail_stage_stat(path, *args, **kwargs): + if path == reservation._stage_directory_name: + if replacement != "none": + stage.rename(moved) + if replacement in ("empty", "populated"): + stage.mkdir() + if replacement == "populated": + (stage / "foreign").write_bytes(b"another writer's evidence") + stage.chmod(0o500) + elif replacement == "symlink": + stage.symlink_to(moved, target_is_directory=True) + raise failure + return stat(path, *args, **kwargs) + + def track_open(*args, **kwargs): + descriptor = open_descriptor(*args, **kwargs) + descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(artifacts.os, "stat", fail_stage_stat) + monkeypatch.setattr(artifacts.os, "open", track_open) + monkeypatch.setattr( + artifacts.os, "supports_dir_fd", os.supports_dir_fd | {fail_stage_stat, track_open} + ) + monkeypatch.setattr( + artifacts.os, "rmdir", lambda *_args, **_kwargs: pytest.fail("removing unverified entry") + ) + monkeypatch.setattr( + artifacts.os, "fchmod", lambda *_args: pytest.fail("changing unverified permissions") + ) + + with pytest.raises( + artifacts.BookReplayError, match="automatic directory cleanup skipped" + ) as caught: + reservation.__enter__() + assert caught.value.__cause__ is failure + assert repr(str(stage)) in str(caught.value) + assert "possible staging residue" in str(caught.value) + assert "ownership could not be verified" in str(caught.value) + assert len(descriptors) == 2 # Only the parent and lock were opened. + for descriptor in descriptors: + with pytest.raises(OSError): + os.fstat(descriptor) + expected = {stage} if replacement == "none" else {moved} + if replacement not in ("none", "missing"): + expected.add(stage) + assert set(tmp_path.iterdir()) == expected + if replacement in ("empty", "populated"): + assert stage.stat().st_mode & 0o777 == 0o500 + if replacement == "populated": + assert (stage / "foreign").read_bytes() == b"another writer's evidence" + elif replacement == "symlink": + assert stage.is_symlink() and stage.readlink() == moved + if moved.exists(): + assert list(moved.iterdir()) == [] + + +@pytest.mark.parametrize("command", ["run", "minimize", "campaign", "sample"]) +def test_cli_reports_initial_stage_stat_failure_before_candidate( + tmp_path, monkeypatch, capsys, command +): + trace = tmp_path / "trace.jsonl" + _trace(trace) + _forbid_candidate(monkeypatch) + stat = os.stat + stages = [] + + def fail_stage_stat(path, *args, **kwargs): + if isinstance(path, str) and path.startswith(".tracebook-stage-"): + stages.append(path) + raise OSError("injected initial staging stat failure") + return stat(path, *args, **kwargs) + + monkeypatch.setattr(artifacts.os, "stat", fail_stage_stat) + monkeypatch.setattr(artifacts.os, "supports_dir_fd", os.supports_dir_fd | {fail_stage_stat}) + arguments = ( + ["sample", str(tmp_path / "sample")] + if command == "sample" + else _command(command, trace, tmp_path / "report.json", tmp_path / "reduced.jsonl") + ) + assert cli.main(arguments) == 2 + output = capsys.readouterr() + assert output.out == "" + assert output.err.startswith("Error:") and "Traceback" not in output.err + assert "automatic directory cleanup skipped" in output.err + assert len(stages) == 1 + parent = tmp_path / "sample" if command == "sample" else tmp_path + stage = parent / stages[0] + assert repr(str(stage)) in output.err + assert stage.is_dir() and list(stage.iterdir()) == [] + if command == "sample": + assert set(parent.iterdir()) == {stage} + assert set(tmp_path.iterdir()) == {trace, parent} + else: + assert set(tmp_path.iterdir()) == {trace, stage} + + def test_cleanup_failure_still_closes_all_descriptors_and_other_sidecars(tmp_path, monkeypatch): reservation = artifacts.OutputReservation(tmp_path / "report.json") reservation.__enter__()