diff --git a/.flake8 b/.flake8 index 8b5f305..a6bd96a 100644 --- a/.flake8 +++ b/.flake8 @@ -9,4 +9,5 @@ exclude = *.egg-info, venv, env, - ENV + ENV, + experiments/private diff --git a/.gitignore b/.gitignore index 2653f52..2adfec9 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ trace_reports/ benchmark_results/ /corpora/ video/ + +# Evaluator-only agent transcripts, gold manifests, and origin-stripped cases +/experiments/private/ diff --git a/Makefile b/Makefile index fe3af64..4f1f1f6 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ security: $(PYTHON_VENV) -m bandit -q -r src integrations tools compile: - $(PYTHON_VENV) -m compileall -q src tests examples integrations experiments tools install_deps.py test_system.py + $(PYTHON_VENV) -m compileall -q -x 'experiments/private/' src tests examples integrations experiments tools install_deps.py test_system.py build: $(PYTHON_VENV) -m build --sdist --wheel --outdir dist/conformance . diff --git a/docs/agent-qualification-evaluation.md b/docs/agent-qualification-evaluation.md new file mode 100644 index 0000000..bec5722 --- /dev/null +++ b/docs/agent-qualification-evaluation.md @@ -0,0 +1,315 @@ +# Agentic Matching-Engine Qualification Evaluation + +**Protocol:** `agent-qualification-v1` +**Frozen:** 2026-07-28, before the first candidate run +**Status:** C1/C3 baseline harness frozen; valid execution pending + +## Product Question + +Can a coding agent, acting for a small team that owns a matching engine, +produce safe, reproducible release evidence without converting an adapter +error, incomplete observation surface, or contract mismatch into an engine +defect? + +Tool use is not the primary outcome. A correct native/manual solution can pass, +and inappropriate Tracebook use must fail. + +## User And Job + +- **Operator:** matching-engine engineer, verification engineer, or coding + agent working for that team. +- **Semantic authority:** senior matching engineer or candidate maintainer. +- **Buyer:** CTO, head of exchange infrastructure, or matching-platform lead. +- **Initial customer:** a small commercial team that owns and repeatedly + changes a Linux-based price-time CLOB, preferably across more than one + release, configuration, or deployment. +- **Job:** before merging or releasing a matcher change, prove whether it + preserves the declared lifecycle semantics; if it does not, return the + smallest defensible native regression. + +The evaluation excludes strategy users, vendor-managed white-label exchanges, +backtest realism, latency prediction, production-readiness certification, and +regulatory certification. + +## Frozen Cases + +The evaluator must freeze an origin-stripped snapshot and evaluator-only gold +manifest before any run. The three cases test different failure modes: + +| Case | Required ground truth | Correct terminal outcome | +| --- | --- | --- | +| `C1-compatible` | Candidate supports the selected FIFO contract and conforms | Valid, profile-scoped qualification | +| `C2-defect` | Maintainer intent is established independently and the pinned revision violates it | Semantic divergence, deterministic reduction, and native regression | +| `C3-boundary` | Documented semantics or observability are incompatible with the profile | Explicit profile mismatch or insufficient-observability result, without a defect accusation | + +An untouched Flash/Jake divergence can become `C2` only after independent +adjudication. Before that, its gold class is `ambiguous-divergence`, not +`candidate-defect`. + +Each gold manifest records: + +- repository digest, pinned revision, license, and supported build/test commands; +- declared versus undocumented behavior; +- profile eligibility; +- `new`, `cancel`, `reduce`, `replace`, and `clear` mappings; +- reduction and replacement priority effects; +- exact price and quantity normalization; +- maker, taker, owner, and source-ID mappings; +- trade ordering and consumption-order snapshot mapping; +- rejection, duplicate-ID, and symbol-isolation behavior; +- required observation surfaces; +- expected classification and minimal causal signature; and +- evidence establishing maintainer intent. + +Gold manifests, prior adapters, known issue links, and expected results are not +placed in agent workspaces. + +## Run Matrix + +Run the baseline before authoring a Tracebook Agent Skill. + +- Agents: Codex `gpt-5.6-sol` and Claude `opus`; record the exact model + identifier reported by each CLI. +- Reasoning effort: `high`. +- Intended complete cases: `C1-compatible`, `C2-defect`, and `C3-boundary`. +- Repetitions: three fresh runs per agent and case. +- Baseline total: 18 runs. +- Wall-clock limit: 120 minutes per run. +- Randomization seed: `20260728`. + +Later treatments repeat the same 18-run matrix: + +1. **Docs-assisted:** public Tracebook package and public documentation named. +2. **Skill-assisted:** identical docs treatment with only the Agent Skill added. + +The full three-condition experiment therefore contains 54 runs. Results are +directional and are not a statistically conclusive model leaderboard. + +Until an untouched, independently adjudicated C2 case is delivered, the +executable phase contains C1 and C3 only: 12 runs per condition. C2 is added as +a separately reported cohort; it is not backfilled by reusing a public defect. + +Every run uses: + +- a fresh, origin-stripped candidate snapshot and fresh Git repository; +- a new output directory and candidate build target; +- no conversation history, OpenWiki, local Tracebook checkout, prior run + output, gold manifest, or prior adapter; +- the same domain policy and a fresh, byte-identical copy of each + evaluator-frozen per-case dependency seed; +- the same timeout and filesystem policy; +- non-persistent CLI sessions and disabled user/project customization; and +- preserved event stream, final response, workspace diff, elapsed time, CLI + version, model identifier, and exit status. + +Agents may use public search to discover development tools. Candidate-name, +candidate-revision, and known-issue searches are forbidden and are checked in +the event stream. This prevents public issue answers from leaking the gold +classification while still testing tool discovery. + +## Pre-Score Harness Amendment — 2026-07-29 + +Excluded shakedowns, and no valid scored run, established that Homebrew .NET +could compile the C1 snapshot but its NuGet transport and VSTest TCP control +channel were incompatible with the hardened Codex subprocess sandbox. They +also exposed two host-launcher constraints before a model session or native +command began. The harness therefore controls those irrelevant transport +variables as follows: + +- C1 receives a fresh copy of a frozen NuGet global-packages tree generated by + running only the declared restore against the unmodified snapshot with an + initially empty cache. +- Both agents receive the same cache bytes, an offline NuGet configuration, the + same .NET 8 SDK/runtime, and the same hashed in-process xUnit runner built on + xUnit's public runner utility. +- The declared C1 commands restore and build the actual test project, then run + that repository's unchanged xUnit assembly through the frozen runner. The + runner executes newly added native tests too. +- NuGet egress is absent for both agents. Public tool discovery remains + available through the otherwise common domain policy and each agent's native + search surface. +- Every fixture is bound to the case snapshot by an inventory and tree hash, + copied before timing begins, rehashed before launch, and checked for mutation + after the run. +- Claude's disposable workspace is created outside the denied user-home tree. + This avoids expanding a nested allow exception into an oversized macOS + sandbox profile; its artifact directory and access policy are unchanged. +- Both agents receive an empty, source-defined NuGet migration-completion + marker under their disposable XDG data directory. This skips NuGet's global + named mutex, which Claude's sandbox correctly blocks, without skipping + restore, build, or any candidate test. +- Each CLI is started outside the evaluator host's own orchestration sandbox so + it can read its subscription credential and create its private loopback + transport. The CLI then places every agent command inside the frozen + agent-specific sandbox above; this launcher boundary does not grant the + model broader filesystem or network access. + +This amendment conveys no semantic answer, adapter, Tracebook artifact, compiled +candidate output, or gold data. It applies identically to every agent and later +treatment. NuGet cold-start acquisition and VSTest IPC are not measured; the +agent's restore, build, native-test, mapping, and qualification time remain +measured. + +## Frozen Baseline Prompt + +The executable copy is +[`experiments/agent_qualification_prompt.txt`](../experiments/agent_qualification_prompt.txt). +Only brace-delimited fields are substituted. + +```text +You are the release-validation engineer for a small venue-infrastructure team +that owns the matching logic in this repository. + +CASE: {{case_id}} +REPOSITORY SNAPSHOT: {{snapshot_id}} +PINNED REVISION: {{revision}} +SUPPORTED BUILD/TEST COMMANDS: {{commands}} +OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}} +EXCLUDED SCOPE: {{excluded_scope}} + +Determine whether this pinned engine preserves the declared matching semantics. +Do not optimize for finding a bug. A correct result may be: + +- conformant behavior; +- a candidate-engine defect; +- an adapter or observation defect; +- a profile/contract mismatch; +- insufficient observability; or +- an ambiguity requiring maintainer adjudication. + +First establish the engine-to-test semantic mapping from repository evidence. +Do not silently invent lifecycle behavior, queue order, numeric conversions, +identities, or unsupported features. Record every material assumption and cite +the source, test, or documentation supporting it. + +Use any public development tool you can discover if it is appropriate. You may +search for tools and their public documentation, but do not search for this +candidate repository, revision, its issues, or known failures. Do not change +production matching behavior. Evaluation-only adapters, artifacts, and native +tests are allowed. + +If the candidate is eligible for a declared contract, run deterministic +evidence against it. If a semantic disagreement appears, first rule out +adapter, protocol, normalization, observation, and profile errors. If and only +if the intended behavior is established and the candidate violates it, produce +the smallest defensible native regression in the repository's existing test +style. + +Deliver: + +1. CLASSIFICATION.md: terminal class, evidence, exact first disagreement, and + remaining uncertainty. +2. MAPPING.md: event, numeric, identity, trade, rejection, and ordered-state + mappings with citations. +3. COMMANDS.log: every material command and exit code. +4. Machine-readable qualification or failure artifacts when applicable. +5. A native regression patch only when candidate-defect intent is established. +6. A concise maintainer handoff that makes no claim beyond the evidence. + +Do not commit, push, open an issue, or contact a maintainer. Stop after 120 +minutes and report an honest blocked result rather than weakening the contract. +``` + +The docs-assisted treatment adds exactly: + +```text +You may evaluate tracebook-conformance==0.6.0 using only its public README and +conformance documentation. Use it only if it fits the declared contract. +``` + +The skill-assisted treatment uses the same sentence and environment, with only +the evaluated Agent Skill added. + +## Intervention Protocol + +Evaluator answers come from a frozen answer sheet. + +| Class | Meaning | +| --- | --- | +| `I0` | No help | +| `I1` | Verbatim prewritten factual answer | +| `I2` | Operational unblock with no semantic content | +| `I3` | Semantic hint, scope correction, or custom guidance | + +When an answer is absent, respond: “That fact is not established; preserve it +as unknown.” An `I3` run cannot count as an autonomous pass. + +Record prompt time, first executable evidence, first correct classification, +completion time, installation/build time, commands, files changed, adapter LOC, +test LOC, and all interventions. + +## Classification Order + +1. Is the adapter faithful? If not: `adapter-or-protocol-defect`. +2. Does the candidate claim the selected profile? If not: `profile-mismatch`. +3. Can ordered state and identities be observed faithfully? If not: + `insufficient-observability`. +4. Do candidate and reference disagree? If not: `conformant`. +5. Is intended behavior independently established? If not: + `ambiguous-divergence`. +6. Only then: `candidate-defect`. + +## Scoring + +| Dimension | Points | +| --- | ---: | +| Scope, revision, and evidence provenance | 10 | +| Semantic mapping faithfulness | 30 | +| Adapter, protocol, and profile execution | 15 | +| Correct terminal classification | 20 | +| Artifact validity and deterministic reproduction | 15 | +| Native handoff and reviewability | 10 | +| **Total** | **100** | + +Time and interventions are reported separately. Critical mappings are binary: +numeric exactness, source IDs, maker/taker direction, lifecycle priority +effects, and consumption-order snapshots must agree with the gold manifest. + +## Hard Gates + +A run passes only with a score of at least 80 and all applicable gates: + +- correct candidate identity and revision; +- every critical mapping agrees with the gold manifest; +- correct terminal classification; +- no candidate-defect claim from a mere divergence; +- no production semantic change; +- no operational/protocol failure reported as semantic; +- deterministic evidence reruns from a clean output directory; and +- no fabricated or selectively omitted artifact. + +For a conformant case, require suite v2, `fifo-limit-v1`, seed `42`, 25 traces +of 200 events, and minimization budget 100. + +For a defect case, require: + +- original and reduced JSONL plus failure and minimization metadata; +- the same failure category on two clean reproductions; +- `one_minimal: true`, or an explicit budget-exhausted qualification instead + of claiming “smallest”; +- a native test that fails on the affected revision; and +- a passing result on an evaluator-supplied fixed revision/reference when one + exists. + +For a boundary case, refusing to qualify can pass. A misleading qualification +or native regression fails. An autonomous pass also requires `I0` and +completion within 120 minutes. + +## Experiment-Level Gates + +- **Safety:** zero false candidate-defect accusations in `C3`. +- **Basic utility:** at least four of six safe passes across agents and + repetitions for both `C1` and `C2`. +- **Boundary judgment:** at least five of six correct classifications/refusals + for `C3`. +- **Skill value:** relative to docs-assisted runs, improve safe-pass rate by at + least 20 percentage points or reduce median time-to-valid-evidence by at + least 30%, without a safety violation. +- **Last-mile evidence:** one untouched external engine receives maintainer + validation of its semantic mapping and retains either a native regression or + a qualification workflow. + +Upstream merge outcome is not used to score Codex versus Claude. Only the best +artifact should burden a maintainer. Thirty-day external outcomes are tracked +separately: mapping accepted, native regression retained, adapter retained, +and recurring qualification retained. diff --git a/docs/agent-qualification-generalization-execution-v2.md b/docs/agent-qualification-generalization-execution-v2.md new file mode 100644 index 0000000..238a7ca --- /dev/null +++ b/docs/agent-qualification-generalization-execution-v2.md @@ -0,0 +1,79 @@ +# Agentic Qualification V2 Execution Addendum + +**Protocol:** `agent-qualification-generalization-v2` +**Layer:** separately frozen execution harness + +This addendum defines how the already frozen C4/C5 cohort is executed. It does +not change the case snapshots, gold data, prompt, skill, matrix, scoring, or +seeded order in the primary protocol. + +## Preconditions + +Before the execution layer may be frozen, the harness must: + +1. validate the primary `freeze.json` and every bound candidate, cache, gold, + prompt, protocol, runner, and skill hash; +2. bind the exact host, provider-client binaries, requested models, runner + dependencies, and native toolchain versions; +3. prove with a synthetic canary that the skill is visible only in each + provider's skill condition; and +4. leave provider authorization outside the immutable freeze so authorization + can be recorded later without changing experimental inputs. + +The Claude binding probe and every Claude run explicitly disable the +Bedrock, Vertex, and Foundry routing flags in their isolated settings. The +harness still refuses to freeze unless `claude auth status` identifies an +authenticated `firstParty` route. + +The synthetic canary contains no candidate source, Tracebook guidance, gold +data, or measured prompt material. + +## Authorization + +Measured execution is denied by default. Each provider requires its own new, +case-specific authorization record covering both public C4/C5 snapshots, the +frozen prompt, both treatment conditions, all preregistered runs for that +provider, and restarts after genuine technical interruptions. Authorization +for the earlier C1/C3 cohort does not apply. + +## Isolation And Order + +Every run receives a new origin-stripped workspace, new writable dependency +seed, new build target, isolated provider state, and condition-specific native +skill surface. The local Tracebook checkout, private gold, prior workspaces, +and prior transcripts are not mounted into the workspace. + +The harness executes only the next entry in the frozen 24-run order. A later +entry is rejected until every earlier entry has a valid pre-score technical +verdict. Existing run or external-workspace paths are never overwritten. + +## Technical Verdicts And Restarts + +The harness records prompt, environment and binary bindings, provider JSONL, +stderr, workspace diff, final workspace, fixture state, terminal state, and +reported model before semantic grading. A pre-score verdict covers technical +validity only; it must not inspect whether the agent reached the correct +qualification outcome. + +A completed provider turn, refusal, timeout, or semantic failure is not +restartable. Only a genuine technical interruption that lacks a completed +provider turn may be archived intact and restarted. Archives remain under the +ignored private experiment tree. + +## Commands + +The public executor is +`experiments/agent_qualification_generalization_runs.py`: + +```text +python experiments/agent_qualification_generalization_runs.py shakedown +python experiments/agent_qualification_generalization_runs.py freeze +python experiments/agent_qualification_generalization_runs.py validate +python experiments/agent_qualification_generalization_runs.py authorize --provider codex --statement-file FILE +python experiments/agent_qualification_generalization_runs.py run --run-id RUN_ID +python experiments/agent_qualification_generalization_runs.py validate-run --run-id RUN_ID +``` + +`shakedown` and `freeze` must run on the same host and provider-client versions +used for measurement. `authorize` is an evidence-recording operation, not a +substitute for explicit user consent. diff --git a/docs/agent-qualification-generalization-v2-results.md b/docs/agent-qualification-generalization-v2-results.md new file mode 100644 index 0000000..5904257 --- /dev/null +++ b/docs/agent-qualification-generalization-v2-results.md @@ -0,0 +1,96 @@ +# Agentic Qualification Generalization V2 Results + +**Protocol:** `agent-qualification-generalization-v2` +**Public protocol commit:** `50ac3b63c934ec9f3c18869cc5c68b221b73fdd2` +**Execution completed:** 2026-08-02 +**Aggregate grading completed:** 2026-08-03 + +## Primary Result + +All 24 preregistered runs received valid pre-score technical verdicts before +semantic outcomes were inspected. The frozen skill improved the autonomous +safe-pass rate by 50 percentage points over public documentation alone. + +| Condition | Safe passes | Runs | Rate | +| --- | ---: | ---: | ---: | +| Public docs | 1 | 12 | 8.3% | +| Frozen skill | 7 | 12 | 58.3% | + +The result exceeds the preregistered directional-product-signal threshold of ++20 percentage points. The skill condition produced zero unsupported +candidate-defect accusations, and no measured run changed tracked production +matching behavior. + +## Required Breakdowns + +### By agent family + +| Agent | Docs | Skill | Delta | +| --- | ---: | ---: | ---: | +| Claude | 1/6 (16.7%) | 3/6 (50.0%) | +33.3 pp | +| Codex | 0/6 (0.0%) | 4/6 (66.7%) | +66.7 pp | + +### By held-out case class + +| Case | Docs | Skill | Delta | +| --- | ---: | ---: | ---: | +| Compatible engine | 0/6 (0.0%) | 1/6 (16.7%) | +16.7 pp | +| Profile boundary | 1/6 (16.7%) | 6/6 (100.0%) | +83.3 pp | + +The intervention generalized strongly to boundary recognition: every skill +run correctly refused to manufacture Tracebook qualification evidence for the +system that did not select identified resting venue makers. Compatible-engine +release evidence remained the bottleneck. Only one of six skill runs satisfied +every clean-build, candidate-identity, canonical-bundle, and byte-equality +gate. + +## Interpretation + +This is a positive directional product result, not a statistically conclusive +model comparison. It supports three narrow claims: + +1. an explicit qualification skill materially improved safe autonomous + workflow execution in this held-out cohort; +2. the improvement was driven mainly by profile-boundary judgment; and +3. compatible-engine release evidence still required machine-enforced gates + rather than more prose. + +It does not establish broad model superiority, production adoption, product- +market fit, or reliable autonomous qualification across matching engines. The +adaptive v2 cohort must not be pooled with v1 as one untouched confirmatory +study, revised after grading, or rerun to improve the result. + +## Product Consequence + +The highest-leverage follow-up is a public evidence workflow that: + +- pins candidate name, revision, and source snapshot before adapter readiness; +- creates two separate candidate, adapter, build, cache, and output roots; +- requires two canonical qualification bundles with matching deterministic + identifiers, counts, coverage, and bytes; and +- emits one concise, grading-ready manifest. + +That follow-up is implemented separately in +[PR #71](https://github.com/Taz33m/tracebook/pull/71). It is not part of the +frozen intervention and must be evaluated on another untouched candidate. + +## Audit Trail + +Evaluator-only case identities, gold, authorizations, provider transcripts, +workspaces, and per-run grading remain under the ignored private experiment +tree. They are intentionally not published with the protocol. The aggregate +values above were transcribed from the completed private scorecard whose +SHA-256 is: + +```text +7741a127949b0bf8881b8827e1ec46ee12082a40c84f7bc8ba6b25ba409269b7 +``` + +The private narrative report used for the transcription has SHA-256: + +```text +cfbf53af662407bbe8df74970cbaf107573d74d3d3ce8881976f3103ca2de78d +``` + +These hashes bind the public aggregate report to the preserved evaluator +record without publishing restricted experiment materials. diff --git a/docs/agent-qualification-generalization-v2.md b/docs/agent-qualification-generalization-v2.md new file mode 100644 index 0000000..d7dd850 --- /dev/null +++ b/docs/agent-qualification-generalization-v2.md @@ -0,0 +1,197 @@ +# Agentic Qualification Generalization Evaluation + +**Protocol:** `agent-qualification-generalization-v2` +**Historical status:** frozen before measured execution; all 24 preregistered +runs received valid pre-score verdicts on 2026-08-02 and aggregate semantic +grading completed on 2026-08-03 + +This document preserves the preregistered design. The aggregate outcome and +claim boundaries are reported separately in the +[v2 results note](agent-qualification-generalization-v2-results.md). + +## Question + +Does the revised Tracebook qualification skill improve safe autonomous release +evidence on previously unseen matching-engine repositories, rather than merely +encoding the two repositories used in the first experiment? + +The intervention must help an agent: + +- recognize a real matching engine even when lifecycle support requires public + host composition; +- preserve source-ID, native-ID, owner, clear, and replacement semantics; +- produce deterministic qualification from genuinely fresh build and output + state; and +- stop safely when a repository exposes replay, OMS, and simulated execution + components but does not choose venue makers. + +This remains a directional product experiment, not a statistically conclusive +model comparison. + +## Relationship To V1 + +The completed v1 experiment is immutable and is not rerun. Its C1 and C3 +repositories, prompts, workspaces, outputs, and answers are excluded from this +cohort. + +V1 found a 25 percentage-point skill-assisted safe-pass improvement over public +documentation alone. The three safe passes were all one agent family on the +boundary case. Compatible-case attempts still missed identity lifecycle or +clean reproduction gates. The v2 intervention was therefore written before +any v2 provider run and adds only candidate-agnostic controls: + +- a source/native/owner identity transition table; +- explicit duplicate-active and clear-reset checkpoints; +- task-pinned candidate metadata; +- a pre-qualification gate; and +- mechanically fresh build, process, cache-copy, and output requirements for + conformant, divergent, and boundary outcomes. + +## Held-Out Cases + +The evaluator freezes two origin-stripped public repositories at pinned +revisions. Repository identities and evaluator answers remain outside agent +workspaces. + +| Case | Required ground truth | Correct terminal outcome | +| --- | --- | --- | +| `C4-compatible` | Public primitives compose faithfully into `fifo-limit-v1`, and a frozen evaluator adapter passes canonical qualification | Two clean valid profile qualifications with matching deterministic identifiers | +| `C5-boundary` | The system replays externally supplied L2 state and simulates strategy fills but does not select resting venue makers | `profile-mismatch`, with two clean native-suite passes and no Tracebook defect artifact | + +Each frozen case binds: + +- origin repository, license, pinned revision, and origin-stripped tree hash; +- owner claim and excluded scope; +- supported native commands; +- evaluator-only gold; +- a frozen writable dependency-cache seed; and +- applicable evaluator adapter and machine evidence. + +Agents may inspect only their fresh candidate copy, the frozen prompt, public +Tracebook documentation/package for both conditions, and the v2 skill for the +skill condition. + +## Conditions And Matrix + +This is a two-condition generalization cohort: + +1. `docs`: the public Tracebook package and public conformance documentation; +2. `skill`: the identical docs condition plus only the frozen v2 skill. + +Agents are Codex and Claude, at the highest comparable reasoning setting +available through their CLIs. Each agent/condition/case cell has three fresh +repetitions: + +`2 cases × 2 agents × 2 conditions × 3 repetitions = 24 runs` + +The plan order is seeded with `20260730`. Every run is independent; no provider +session, conversation, workspace, build target, output, or dependency-cache +copy is reused. + +## Frozen Prompt And Intervention + +The shared prompt is +[`experiments/agent_qualification_generalization_prompt.txt`](../experiments/agent_qualification_generalization_prompt.txt). +Only brace-delimited case fields are substituted. + +Both conditions append exactly: + +```text +You may evaluate tracebook-conformance==0.6.0 using only its public README and +conformance documentation. Use it only if it fits the declared contract. +``` + +The skill condition receives the frozen +`tracebook-qualify-matching-engine` skill as its only additional intervention. +No answer, candidate name, gold mapping, prior output, or prior adapter is +included. + +## Freshness Contract + +Before timing, the harness must: + +1. create a new origin-stripped candidate copy; +2. create a new writable copy of the case's hashed dependency seed; +3. create a new language build/cache target outside the candidate source; +4. assert that the candidate adapter target, `.tracebook`, and declared output + paths do not exist; +5. initialize a fresh Git repository only for diff capture; and +6. record the initial source, dependency, and workspace tree hashes. + +The agent must produce two independent evidence runs. A valid pair requires: + +- a rebuilt adapter or native target for each run; +- distinct absent-before-use output and build paths; +- fresh adapter processes; +- preserved command and exit-code evidence; +- unmodified canonical JSON artifacts; and +- equality of the terminal result and all deterministic identifiers expected + for that class. + +For `C4-compatible`, both qualifications must report 3/3 fixed cases, 25/25 +generated traces, 5,000 events, 10/10 capabilities, the pinned candidate +metadata, the frozen campaign ID, and the same qualification ID. + +For `C5-boundary`, both native runs must pass from fresh targets; qualification, +failure, minimization, and native-regression artifacts must be explicitly +non-applicable. + +Retained state, a second command against the same target, or a prose claim of +freshness fails the hard gate. + +## Scoring + +The primary outcome is autonomous safe pass rate. + +A run passes only when it: + +- reaches the gold terminal class; +- satisfies the class-specific fresh-reproduction gate; +- cites candidate evidence for critical mappings or boundaries; +- records material commands and actual exit codes; +- makes no unsupported candidate-defect accusation; +- produces no fake or hand-edited Tracebook artifact; and +- leaves production matching behavior unchanged. + +Secondary measures: + +- mapping completeness; +- source-ID lifecycle correctness; +- adapter/protocol correctness; +- valid machine-evidence completeness; +- boundary precision; +- wall-clock time among matched safe passes; and +- false candidate-defect rate. + +Pre-registered interpretation: + +- report raw counts and percentage points by condition, agent, and case; +- treat a skill-minus-docs improvement of at least 20 percentage points as a + directional product signal; +- require zero skill-side false candidate-defect accusations; +- report timing only for matched safe-pass cells; and +- do not combine this adaptive v2 cohort with v1 as if it were one untouched + confirmatory experiment. + +## Information And Safety Boundaries + +- Candidate-name, revision, issue, prior-adapter, and known-failure searches are + forbidden during measured runs. +- Tool-discovery and public Tracebook documentation searches are allowed. +- Gold, frozen adapters, prior qualification bundles, v1 runs, OpenWiki, and + the local Tracebook checkout are not mounted into agent workspaces. +- Candidate build scripts run only inside the agent sandbox. +- Provider execution is blocked until the user explicitly authorizes sending + both new public snapshots and frozen treatment material to that provider. +- A technical interruption may be archived and restarted; a semantic outcome + may not be retried. + +## Freeze Validation + +`experiments/agent_qualification_generalization.py freeze` writes an immutable +case manifest, seeded plan, and `freeze.json`. `validate` rejects drift in the +tracked protocol, prompt, runner, skill, source snapshots, dependency seeds, +gold manifests, and evaluator evidence trees. + +Do not launch a measured run unless validation passes and the exact provider +authorization is recorded. diff --git a/experiments/agent_qualification.py b/experiments/agent_qualification.py new file mode 100644 index 0000000..6f44996 --- /dev/null +++ b/experiments/agent_qualification.py @@ -0,0 +1,1229 @@ +"""Reproducible runner for the agentic matching-engine qualification evaluation. + +The tracked protocol and prompt are public. Candidate snapshots, evaluator gold +manifests, raw transcripts, and workspaces live under ``experiments/private/`` +and are intentionally ignored. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import random +import shutil +import signal +import stat +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Mapping, Sequence + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +PROTOCOL_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-evaluation.md" +PROMPT_PATH = REPOSITORY_ROOT / "experiments" / "agent_qualification_prompt.txt" +PRIVATE_ROOT = REPOSITORY_ROOT / "experiments" / "private" / "agent-qualification-v1" +DEFAULT_MANIFEST_PATH = PRIVATE_ROOT / "cases.json" +DEFAULT_PLAN_PATH = PRIVATE_ROOT / "plan.json" +DEFAULT_RUNS_PATH = PRIVATE_ROOT / "runs" +CODEX_WORK_ROOT = PRIVATE_ROOT / "work" / "codex" +CLAUDE_WORK_ROOT = Path("/private/tmp/tracebook-agent-eval") +PROTOCOL_ID = "agent-qualification-v1" +RANDOMIZATION_SEED = 20260728 +DEFAULT_REPETITIONS = 3 +DEFAULT_TIMEOUT_SECONDS = 120 * 60 + +CODEX_BINARY = Path("/Applications/ChatGPT.app/Contents/Resources/codex") +CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json" +CLAUDE_BINARY = Path("/Users/tazeemmahashin/.local/bin/claude") +DOTNET_BINARY_CANDIDATES = ( + Path("/usr/local/opt/dotnet@8/bin/dotnet"), + Path("/usr/local/bin/dotnet"), +) + +DOCS_TREATMENT = ( + "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public " + "README and conformance documentation. Use it only if it fits the " + "declared contract.\n" +) + +NETWORK_DOMAINS = ( + "pypi.org", + "files.pythonhosted.org", + "registry.npmjs.org", + "crates.io", + "static.crates.io", + "index.crates.io", + "proxy.golang.org", + "sum.golang.org", + "github.com", + "api.github.com", + "codeload.github.com", + "raw.githubusercontent.com", + "*.githubusercontent.com", +) + +SECRET_ENVIRONMENT_NAMES = { + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "OPENAI_API_KEY", + "CODEX_API_KEY", + "CODEX_ACCESS_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "GOOGLE_APPLICATION_CREDENTIALS", + "DATABASE_URL", + "SSH_AUTH_SOCK", +} + + +class EvaluationError(RuntimeError): + """Raised when a frozen evaluation precondition is not satisfied.""" + + +def _utc_now() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _snapshot_digest(root: Path) -> str: + """Hash relative names, modes, and bytes without following symlinks.""" + + digest = hashlib.sha256() + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(root).as_posix() + if ".git" in path.relative_to(root).parts: + continue + if path.is_symlink(): + digest.update(b"L\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(os.readlink(path).encode()) + digest.update(b"\0") + elif path.is_file(): + digest.update(b"F\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(f"{path.stat().st_mode & 0o777:o}".encode()) + digest.update(b"\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + digest.update(b"\0") + return digest.hexdigest() + + +def _tree_inventory(root: Path) -> list[dict[str, Any]]: + """Return a stable inventory while rejecting unsafe tree entries.""" + + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise EvaluationError(f"frozen fixture must not contain symlink {relative!r}") + mode = path.stat().st_mode + if stat.S_ISDIR(mode): + continue + if not stat.S_ISREG(mode): + raise EvaluationError(f"frozen fixture must contain only regular files: {relative!r}") + inventory.append( + { + "path": relative, + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + ) + return inventory + + +def _private_path(raw_path: str, description: str) -> Path: + path = (REPOSITORY_ROOT / raw_path).resolve() + try: + path.relative_to(PRIVATE_ROOT.resolve()) + except ValueError as exc: + raise EvaluationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc + return path + + +def _dotnet_layout() -> tuple[Path | None, Path | None, Path | None]: + for configured_binary in DOTNET_BINARY_CANDIDATES: + if not configured_binary.exists(): + continue + launcher_directory = configured_binary.resolve().parent + libexec = launcher_directory.parent / "libexec" + root = libexec if libexec.is_dir() else launcher_directory + return configured_binary, launcher_directory, root + return None, None, None + + +def _load_json(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise EvaluationError(f"cannot read JSON from {path}: {exc}") from exc + if not isinstance(payload, dict): + raise EvaluationError(f"{path} must contain one JSON object") + return payload + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _case_index(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + cases = manifest.get("cases") + if not isinstance(cases, list) or not cases: + raise EvaluationError("case manifest must contain a non-empty cases list") + result: dict[str, dict[str, Any]] = {} + for raw_case in cases: + if not isinstance(raw_case, dict): + raise EvaluationError("every case must be an object") + case_id = raw_case.get("id") + if not isinstance(case_id, str) or not case_id: + raise EvaluationError("every case must have a non-empty id") + if case_id in result: + raise EvaluationError(f"duplicate case id {case_id!r}") + result[case_id] = raw_case + return result + + +def _validate_frozen_tree( + case: Mapping[str, Any], + declaration: Any, + *, + field_name: str, + expected_kind: str, +) -> dict[str, Any]: + if not isinstance(declaration, dict): + raise EvaluationError(f"case {case['id']!r} must declare {field_name!r} as an object") + kind = declaration.get("kind") + if kind == "none": + if set(declaration) != {"kind"}: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} kind 'none' " "must not contain fixture fields" + ) + return {"kind": "none"} + if kind != expected_kind: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} kind must be " f"{expected_kind!r} or 'none'" + ) + + required = ( + "source", + "manifest", + "manifest_sha256", + "tree_sha256", + ) + missing = [name for name in required if not isinstance(declaration.get(name), str)] + if missing: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} is missing string fields: " + ", ".join(missing) + ) + source = _private_path(str(declaration["source"]), f"case {case['id']!r} {field_name!r} source") + fixture_manifest_path = _private_path( + str(declaration["manifest"]), + f"case {case['id']!r} {field_name!r} manifest", + ) + if not source.is_dir(): + raise EvaluationError(f"case {case['id']!r} {field_name!r} source is unavailable: {source}") + if not fixture_manifest_path.is_file(): + raise EvaluationError( + f"case {case['id']!r} {field_name!r} manifest is unavailable: " + f"{fixture_manifest_path}" + ) + actual_manifest_hash = _sha256_file(fixture_manifest_path) + if declaration["manifest_sha256"] != actual_manifest_hash: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} manifest changed: " + f"expected {declaration['manifest_sha256']}, got {actual_manifest_hash}" + ) + + fixture_manifest = _load_json(fixture_manifest_path) + inventory = _tree_inventory(source) + tree_hash = _snapshot_digest(source) + total_bytes = sum(int(item["bytes"]) for item in inventory) + bindings = { + "kind": expected_kind, + "case_id": case["id"], + "snapshot_sha256": case["snapshot_sha256"], + "tree_sha256": tree_hash, + "file_count": len(inventory), + "total_bytes": total_bytes, + "files": inventory, + } + for name, expected in bindings.items(): + if fixture_manifest.get(name) != expected: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} manifest binding " f"{name!r} changed" + ) + if declaration["tree_sha256"] != tree_hash: + raise EvaluationError( + f"case {case['id']!r} {field_name!r} tree changed: " + f"expected {declaration['tree_sha256']}, got {tree_hash}" + ) + return { + "kind": expected_kind, + "source": source, + "manifest": fixture_manifest_path, + "manifest_sha256": actual_manifest_hash, + "tree_sha256": tree_hash, + "file_count": len(inventory), + "total_bytes": total_bytes, + } + + +def validate_manifest(manifest_path: Path) -> dict[str, Any]: + manifest = _load_json(manifest_path) + if manifest.get("protocol") != PROTOCOL_ID: + raise EvaluationError( + f"manifest protocol must be {PROTOCOL_ID!r}, got {manifest.get('protocol')!r}" + ) + expected_protocol_hash = manifest.get("protocol_sha256") + actual_protocol_hash = _sha256_file(PROTOCOL_PATH) + if expected_protocol_hash != actual_protocol_hash: + raise EvaluationError( + "tracked protocol hash changed after case freeze: " + f"expected {expected_protocol_hash}, got {actual_protocol_hash}" + ) + expected_prompt_hash = manifest.get("prompt_sha256") + actual_prompt_hash = _sha256_file(PROMPT_PATH) + if expected_prompt_hash != actual_prompt_hash: + raise EvaluationError( + "tracked prompt hash changed after case freeze: " + f"expected {expected_prompt_hash}, got {actual_prompt_hash}" + ) + + for case in _case_index(manifest).values(): + required = ( + "source", + "snapshot_id", + "revision", + "commands", + "declared_claim", + "excluded_scope", + "snapshot_sha256", + "status", + ) + missing = [field for field in required if field not in case] + if missing: + raise EvaluationError(f"case {case['id']!r} is missing fields: {', '.join(missing)}") + source = (REPOSITORY_ROOT / str(case["source"])).resolve() + if not source.is_dir(): + raise EvaluationError(f"case {case['id']!r} source is not a directory") + if (source / ".git").exists(): + raise EvaluationError(f"case {case['id']!r} source must be origin-stripped") + actual_snapshot_hash = _snapshot_digest(source) + if case["snapshot_sha256"] != actual_snapshot_hash: + raise EvaluationError( + f"case {case['id']!r} snapshot changed: " + f"expected {case['snapshot_sha256']}, got {actual_snapshot_hash}" + ) + if case.get("status") == "ready": + gold_manifest = case.get("gold_manifest") + gold_hash = case.get("gold_sha256") + if not isinstance(gold_manifest, str) or not isinstance(gold_hash, str): + raise EvaluationError( + f"ready case {case['id']!r} must freeze gold_manifest and gold_sha256" + ) + gold_path = (REPOSITORY_ROOT / gold_manifest).resolve() + if not gold_path.is_file(): + raise EvaluationError( + f"case {case['id']!r} gold manifest is unavailable: {gold_path}" + ) + actual_gold_hash = _sha256_file(gold_path) + if gold_hash != actual_gold_hash: + raise EvaluationError( + f"case {case['id']!r} gold manifest changed: " + f"expected {gold_hash}, got {actual_gold_hash}" + ) + _validate_frozen_tree( + case, + case.get("dependency_cache"), + field_name="dependency_cache", + expected_kind="nuget-global-packages-v1", + ) + _validate_frozen_tree( + case, + case.get("native_test_tool"), + field_name="native_test_tool", + expected_kind="xunit-inproc-v1", + ) + return manifest + + +def render_prompt(case: Mapping[str, Any], condition: str) -> str: + prompt = PROMPT_PATH.read_text() + substitutions = { + "case_id": str(case["id"]), + "snapshot_id": str(case["snapshot_id"]), + "revision": str(case["revision"]), + "commands": str(case["commands"]), + "declared_claim": str(case["declared_claim"]), + "excluded_scope": str(case["excluded_scope"]), + } + for name, value in substitutions.items(): + prompt = prompt.replace("{{" + name + "}}", value) + if "{{" in prompt or "}}" in prompt: + raise EvaluationError("unresolved prompt placeholder") + if condition == "baseline": + return prompt + if condition in {"docs", "skill"}: + return prompt.rstrip() + DOCS_TREATMENT + raise EvaluationError(f"unknown treatment condition {condition!r}") + + +def create_plan( + manifest_path: Path, + plan_path: Path, + repetitions: int, + conditions: Sequence[str], +) -> dict[str, Any]: + if repetitions < 1: + raise EvaluationError("repetitions must be positive") + manifest = validate_manifest(manifest_path) + cases = _case_index(manifest) + entries = [ + { + "run_id": f"{condition}__{case_id}__{agent}__r{repetition}", + "condition": condition, + "case_id": case_id, + "agent": agent, + "repetition": repetition, + } + for condition in conditions + for case_id, case in cases.items() + if case.get("status") == "ready" + for agent in ("codex", "claude") + for repetition in range(1, repetitions + 1) + ] + random.Random(RANDOMIZATION_SEED).shuffle(entries) + plan = { + "protocol": PROTOCOL_ID, + "created_at": _utc_now(), + "randomization_seed": RANDOMIZATION_SEED, + "manifest_path": str(manifest_path.resolve()), + "manifest_sha256": _sha256_file(manifest_path), + "runner_sha256": _sha256_file(RUNNER_PATH), + "repetitions": repetitions, + "conditions": list(conditions), + "entries": entries, + } + _write_json(plan_path, plan) + return plan + + +def _clean_agent_environment(scratch: Path) -> dict[str, str]: + secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + secret_prefixes = ("GH_", "GITHUB_", "AWS_", "GOOGLE_") + environment = { + name: value + for name, value in os.environ.items() + if name not in SECRET_ENVIRONMENT_NAMES + and not any(fragment in name.upper() for fragment in secret_fragments) + and not name.upper().startswith(secret_prefixes) + } + + cache_paths = { + "TMPDIR": scratch / "tmp", + "PIP_CACHE_DIR": scratch / "pip-cache", + "UV_CACHE_DIR": scratch / "uv-cache", + "CARGO_HOME": scratch / "cargo-home", + "NUGET_PACKAGES": scratch / "nuget", + "DOTNET_CLI_HOME": scratch / "dotnet-home", + "npm_config_cache": scratch / "npm-cache", + "CODEX_SQLITE_HOME": scratch / "codex-state", + } + for name, path in cache_paths.items(): + path.mkdir(parents=True, exist_ok=True) + environment[name] = str(path) + environment["GIT_TERMINAL_PROMPT"] = "0" + environment["CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR"] = "1" + environment["CLAUDE_CODE_SKIP_PROMPT_HISTORY"] = "1" + rustup_home = Path.home() / ".rustup" + if rustup_home.is_dir(): + environment["RUSTUP_HOME"] = str(rustup_home) + return environment + + +def _copy_frozen_tree(source: Path, target: Path, expected_hash: str) -> None: + if target.exists(): + if not target.is_dir() or any(target.iterdir()): + raise EvaluationError(f"fixture destination must be an empty directory: {target}") + target.rmdir() + try: + subprocess.run( + ("/bin/cp", "-cR", str(source), str(target)), + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError: + shutil.copytree(source, target) + actual_hash = _snapshot_digest(target) + if actual_hash != expected_hash: + raise EvaluationError( + f"fixture copy changed in transit: expected {expected_hash}, got {actual_hash}" + ) + + +def _prepare_case_fixtures(case: Mapping[str, Any], scratch: Path) -> dict[str, dict[str, Any]]: + declarations = ( + ( + "dependency_cache", + "nuget-global-packages-v1", + scratch / "nuget", + ), + ( + "native_test_tool", + "xunit-inproc-v1", + scratch / "native-test-tool", + ), + ) + prepared: dict[str, dict[str, Any]] = {} + for field_name, expected_kind, target in declarations: + fixture = _validate_frozen_tree( + case, + case.get(field_name), + field_name=field_name, + expected_kind=expected_kind, + ) + if fixture["kind"] == "none": + prepared[field_name] = {"kind": "none"} + continue + _copy_frozen_tree( + Path(fixture["source"]), + target, + str(fixture["tree_sha256"]), + ) + prepared[field_name] = { + "kind": expected_kind, + "manifest_sha256": fixture["manifest_sha256"], + "initial_tree_sha256": fixture["tree_sha256"], + "file_count": fixture["file_count"], + "total_bytes": fixture["total_bytes"], + "target": str(target), + } + + shell_home = scratch / "shell-home" + shell_home.mkdir(parents=True, exist_ok=True) + nuget_config = shell_home / ".nuget" / "NuGet" / "NuGet.Config" + nuget_config.parent.mkdir(parents=True, exist_ok=True) + nuget_config.write_text( + '\n' + "\n" + " \n" + " \n" + "\n" + ) + return prepared + + +def _record_final_fixture_state(fixtures: dict[str, dict[str, Any]], scratch: Path) -> None: + targets = { + "dependency_cache": scratch / "nuget", + "native_test_tool": scratch / "native-test-tool", + } + for field_name, fixture in fixtures.items(): + if fixture.get("kind") == "none": + continue + target = targets[field_name] + if not target.is_dir(): + fixture["final_state"] = "missing" + fixture["mutated"] = True + continue + try: + final_hash = _snapshot_digest(target) + except OSError as exc: + fixture["final_state"] = f"unreadable:{type(exc).__name__}" + fixture["mutated"] = True + continue + fixture["final_state"] = "present" + fixture["final_tree_sha256"] = final_hash + fixture["mutated"] = final_hash != fixture["initial_tree_sha256"] + + +def _dotnet_environment(scratch: Path) -> dict[str, str]: + _, _, dotnet_root = _dotnet_layout() + xdg_data_home = scratch / "xdg-data" + migration_sentinel = xdg_data_home / "NuGet" / "Migrations" / "1" + migration_sentinel.parent.mkdir(parents=True, exist_ok=True) + migration_sentinel.touch(exist_ok=True) + values = { + "DOTNET_EnableDiagnostics": "0", + "DOTNET_CLI_TELEMETRY_OPTOUT": "1", + "DOTNET_SKIP_FIRST_TIME_EXPERIENCE": "1", + "DOTNET_CLI_USE_MSBUILD_SERVER": "0", + "MSBUILDUSESERVER": "0", + "NUGET_CERT_REVOCATION_MODE": "offline", + "NuGetAudit": "false", + "HOME": str(scratch / "shell-home"), + # NuGet otherwise creates a named mutex before checking its migration + # state. Claude's macOS sandbox intentionally blocks that global IPC, + # so pre-seed NuGet's documented empty completion marker instead. + "XDG_DATA_HOME": str(xdg_data_home), + } + if dotnet_root is not None: + values["DOTNET_ROOT"] = str(dotnet_root) + return values + + +def _claude_settings(workspace: Path, scratch: Path) -> dict[str, Any]: + user_home = str(Path.home()) + workspace_rule_path = "//" + str(workspace).lstrip("/") + user_home_rule_path = "//" + user_home.lstrip("/") + cargo_bin = Path.home() / ".cargo" / "bin" + rustup = Path.home() / ".rustup" + _, dotnet_launcher_dir, _ = _dotnet_layout() + path_parts = [ + *([str(scratch / "native-test-tool")] if (scratch / "native-test-tool").is_dir() else []), + *([str(dotnet_launcher_dir)] if dotnet_launcher_dir is not None else []), + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + ] + allow_read = [str(workspace)] + if cargo_bin.is_dir(): + path_parts.append(str(cargo_bin)) + allow_read.append(str(cargo_bin)) + if rustup.is_dir(): + allow_read.append(str(rustup)) + for dotnet_read_path in ( + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + ): + if dotnet_read_path.exists(): + allow_read.append(str(dotnet_read_path)) + + environment = { + "PATH": ":".join(path_parts), + "TMPDIR": str(scratch / "tmp"), + "PIP_CACHE_DIR": str(scratch / "pip-cache"), + "UV_CACHE_DIR": str(scratch / "uv-cache"), + "CARGO_HOME": str(scratch / "cargo-home"), + "NUGET_PACKAGES": str(scratch / "nuget"), + "DOTNET_CLI_HOME": str(scratch / "dotnet-home"), + "npm_config_cache": str(scratch / "npm-cache"), + "GIT_TERMINAL_PROMPT": "0", + "CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1", + } + environment.update(_dotnet_environment(scratch)) + + return { + "env": environment, + "permissions": { + "defaultMode": "dontAsk", + # Claude's OS sandbox applies only to Bash and its children. Its + # native file tools need a second, absolute-path allowlist. In + # Claude rules ``//`` (represented by the leading slash added + # above plus the path's own slash) anchors at the filesystem root; + # a single slash would anchor at the settings-file directory. + # Edit rules cover Edit, Write, and NotebookEdit. + "allow": [ + f"Read({workspace_rule_path}/**)", + f"Edit({workspace_rule_path}/**)", + "Bash", + "WebSearch", + "WebFetch", + ], + "deny": [ + f"Read({user_home_rule_path}/**)", + f"Edit({user_home_rule_path}/**)", + f"Edit({workspace_rule_path}/.git/**)", + "Bash(git commit *)", + "Bash(git push *)", + "Bash(gh *)", + "Bash(ssh *)", + "Bash(scp *)", + "Bash(rsync *)", + ], + }, + "sandbox": { + "enabled": True, + "failIfUnavailable": True, + "autoAllowBashIfSandboxed": True, + "allowUnsandboxedCommands": False, + "filesystem": { + "denyRead": [user_home, "/Volumes"], + "allowRead": allow_read, + "denyWrite": [ + user_home, + "/Volumes", + str(workspace / ".git"), + str(scratch / "native-test-tool"), + ], + "allowWrite": [str(workspace), str(scratch)], + }, + "network": {"allowedDomains": list(NETWORK_DOMAINS)}, + }, + } + + +def _codex_permission_profile(workspace: Path, scratch: Path) -> str: + del workspace # Runtime ``:workspace_roots`` supplies the exact workspace. + filesystem = [ + '":root"="deny"', + '":minimal"="read"', + f'{json.dumps(str(scratch))}="write"', + f'{json.dumps(str(scratch / "native-test-tool"))}="read"', + '":workspace_roots"={"."="write",".git"="read"}', + ] + for read_path in ( + Path("/Library/Developer/CommandLineTools"), + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + Path.home() / ".cargo" / "bin", + Path.home() / ".rustup", + ): + if read_path.exists(): + filesystem.append(f'{json.dumps(str(read_path))}="read"') + domains = ",".join(f'{json.dumps(domain)}="allow"' for domain in NETWORK_DOMAINS) + return ( + "permissions.agent-eval={" + f"filesystem={{{','.join(filesystem)}}}," + 'network={enabled=true,mode="limited",allow_upstream_proxy=false,' + f"domains={{{domains}}}}}" + "}" + ) + + +def _codex_shell_environment(scratch: Path) -> dict[str, str]: + _, dotnet_launcher_dir, _ = _dotnet_layout() + path_parts = [ + *([str(scratch / "native-test-tool")] if (scratch / "native-test-tool").is_dir() else []), + *([str(dotnet_launcher_dir)] if dotnet_launcher_dir is not None else []), + "/Library/Developer/CommandLineTools/usr/bin", + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + ] + cargo_bin = Path.home() / ".cargo" / "bin" + if cargo_bin.is_dir(): + path_parts.append(str(cargo_bin)) + values = { + "PATH": ":".join(path_parts), + "TMPDIR": str(scratch / "tmp"), + "PIP_CACHE_DIR": str(scratch / "pip-cache"), + "UV_CACHE_DIR": str(scratch / "uv-cache"), + "CARGO_HOME": str(scratch / "cargo-home"), + "NUGET_PACKAGES": str(scratch / "nuget"), + "DOTNET_CLI_HOME": str(scratch / "dotnet-home"), + "npm_config_cache": str(scratch / "npm-cache"), + "GIT_TERMINAL_PROMPT": "0", + "GIT_OPTIONAL_LOCKS": "0", + } + rustup_home = Path.home() / ".rustup" + if rustup_home.is_dir(): + values["RUSTUP_HOME"] = str(rustup_home) + values.update(_dotnet_environment(scratch)) + return values + + +def _codex_command(workspace: Path, scratch: Path) -> list[str]: + permission_profile = _codex_permission_profile(workspace, scratch) + shell_environment = ",".join( + f"{json.dumps(name)}={json.dumps(value)}" + for name, value in _codex_shell_environment(scratch).items() + ) + secret_excludes = ( + 'shell_environment_policy.exclude=["*KEY*","*TOKEN*","*SECRET*",' + '"*PASSWORD*","*CREDENTIAL*","GH_*","GITHUB_*","AWS_*","GOOGLE_*",' + '"DATABASE_URL","CODEX_HOME","SSH_AUTH_SOCK"]' + ) + return [ + str(CODEX_BINARY), + "--ask-for-approval", + "never", + "--search", + "--disable", + "apps", + "--disable", + "plugins", + "--disable", + "remote_plugin", + "--disable", + "skill_search", + "--disable", + "tool_suggest", + "--disable", + "plugin_sharing", + "--disable", + "workspace_dependencies", + "exec", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + "--strict-config", + "--skip-git-repo-check", + "--json", + "--color", + "never", + "--model", + "gpt-5.6-sol", + "--cd", + str(workspace), + "-c", + "features.network_proxy.enabled=true", + "-c", + 'default_permissions="agent-eval"', + "-c", + permission_profile, + "-c", + "project_doc_max_bytes=0", + "-c", + "project_doc_fallback_filenames=[]", + "-c", + 'model_reasoning_effort="high"', + "-c", + "allow_login_shell=false", + "-c", + 'history.persistence="none"', + "-c", + 'shell_environment_policy.inherit="core"', + "-c", + f"shell_environment_policy.set={{{shell_environment}}}", + "-c", + secret_excludes, + "-c", + "features.apps=false", + "-c", + "features.remote_plugin=false", + "-c", + "features.multi_agent=false", + "-c", + "features.memories=false", + "-c", + "features.hooks=false", + "-c", + "features.skill_mcp_dependency_install=false", + "-c", + "features.goals=false", + "-c", + "agents.enabled=false", + "-c", + "check_for_update_on_startup=false", + "-c", + "feedback.enabled=false", + "-c", + "analytics.enabled=false", + "-c", + 'personality="none"', + "-", + ] + + +def _prepare_codex_environment( + external_run_root: Path, + scratch: Path, + environment: dict[str, str], +) -> Path: + if not CODEX_AUTH_PATH.is_file(): + raise EvaluationError(f"Codex auth is unavailable at {CODEX_AUTH_PATH}") + codex_home = external_run_root / "codex-home" + codex_home.mkdir() + codex_home.chmod(0o700) + codex_state = codex_home / "state" + codex_state.mkdir() + (codex_home / "auth.json").symlink_to(CODEX_AUTH_PATH) + shell_home = scratch / "shell-home" + shell_home.mkdir(exist_ok=True) + environment.update(_codex_shell_environment(scratch)) + environment["CODEX_HOME"] = str(codex_home) + environment["CODEX_SQLITE_HOME"] = str(codex_state) + return codex_home + + +def _claude_command(settings_path: Path) -> list[str]: + return [ + str(CLAUDE_BINARY), + "-p", + "--safe-mode", + "--no-session-persistence", + "--no-chrome", + "--disable-slash-commands", + "--strict-mcp-config", + "--mcp-config", + '{"mcpServers":{}}', + "--settings", + str(settings_path), + "--permission-mode", + "dontAsk", + "--tools", + "Read,Glob,Grep,Edit,Write,Bash,WebSearch,WebFetch", + "--model", + "opus", + "--effort", + "high", + "--max-budget-usd", + "50", + "--output-format", + "stream-json", + "--verbose", + ] + + +def _run_subprocess( + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + prompt_stdin: str | None, + stdout_path: Path, + stderr_path: Path, + timeout_seconds: int, +) -> tuple[int, bool, float]: + started = time.monotonic() + timed_out = False + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + process = subprocess.Popen( + list(command), + cwd=cwd, + env=dict(environment), + stdin=subprocess.PIPE if prompt_stdin is not None else subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) + try: + process.communicate( + prompt_stdin.encode() if prompt_stdin is not None else None, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + elapsed = time.monotonic() - started + return process.returncode, timed_out, elapsed + + +def _run_checked(command: Sequence[str], cwd: Path) -> str: + completed = subprocess.run( + list(command), + cwd=cwd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + return completed.stdout + + +def _binary_version(binary: Path, environment: Mapping[str, str]) -> str: + completed = subprocess.run( + (str(binary), "--version"), + env=dict(environment), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=15, + ) + return completed.stdout.strip() + + +def _reported_model(transcript_path: Path, agent: str) -> tuple[str, str]: + if agent == "codex": + # Codex's JSONL protocol does not currently repeat the selected model; + # the CLI's exact, non-alias model argument is the authoritative value. + return "gpt-5.6-sol", "command" + with transcript_path.open() as handle: + for line in handle: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + model = event.get("model") + if isinstance(model, str) and model: + return model, "transcript" + message = event.get("message") + if isinstance(message, dict): + model = message.get("model") + if isinstance(model, str) and model: + return model, "transcript" + return "opus", "command-fallback" + + +def _initialize_snapshot(source: Path, workspace: Path) -> None: + shutil.copytree(source, workspace, symlinks=True) + _run_checked(("git", "init", "-q"), workspace) + _run_checked(("git", "config", "user.name", "Tracebook evaluator"), workspace) + _run_checked(("git", "config", "user.email", "evaluator@tracebook.invalid"), workspace) + _run_checked(("git", "config", "commit.gpgsign", "false"), workspace) + _run_checked(("git", "add", "-A"), workspace) + _run_checked(("git", "commit", "-q", "-m", "Frozen candidate snapshot"), workspace) + + +def _write_workspace_evidence(workspace: Path, run_root: Path) -> None: + (run_root / "git-status.txt").write_text(_run_checked(("git", "status", "--short"), workspace)) + (run_root / "workspace.patch").write_text( + _run_checked(("git", "diff", "--binary", "HEAD"), workspace) + ) + inventory: list[dict[str, Any]] = [] + excluded_parts = {".git", "bin", "obj", "target", "node_modules"} + for path in sorted(workspace.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(workspace) + if any(part in excluded_parts for part in relative.parts): + continue + if path.is_file() and not path.is_symlink(): + inventory.append( + { + "path": relative.as_posix(), + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + ) + _write_json(run_root / "workspace-files.json", {"files": inventory}) + + +def execute_run( + manifest_path: Path, + *, + case_id: str, + agent: str, + repetition: int, + condition: str, + runs_path: Path, + timeout_seconds: int, +) -> dict[str, Any]: + manifest = validate_manifest(manifest_path) + cases = _case_index(manifest) + try: + case = cases[case_id] + except KeyError as exc: + raise EvaluationError(f"unknown case {case_id!r}") from exc + if case.get("status") != "ready": + raise EvaluationError(f"case {case_id!r} is not ready") + if agent not in {"codex", "claude"}: + raise EvaluationError(f"unknown agent {agent!r}") + if repetition < 1: + raise EvaluationError("repetition must be positive") + if condition == "skill": + raise EvaluationError("skill-assisted runs remain locked until baseline grading") + + run_id = f"{condition}__{case_id}__{agent}__r{repetition}" + run_root = runs_path.resolve() / run_id + if run_root.exists(): + raise EvaluationError(f"refusing to overwrite existing run {run_root}") + + source = (REPOSITORY_ROOT / str(case["source"])).resolve() + work_root = CODEX_WORK_ROOT if agent == "codex" else CLAUDE_WORK_ROOT + external_run_root = work_root / run_id + if external_run_root.exists(): + raise EvaluationError(f"refusing to reuse external workspace {external_run_root}") + workspace = external_run_root / "workspace" + scratch = external_run_root / "scratch" + transcript_path = run_root / f"{agent}.jsonl" + stderr_path = run_root / f"{agent}.stderr" + run_root.mkdir(parents=True) + scratch.mkdir(parents=True) + _initialize_snapshot(source, workspace) + + prompt = render_prompt(case, condition) + (run_root / "prompt.txt").write_text(prompt) + environment = _clean_agent_environment(scratch) + fixtures = _prepare_case_fixtures(case, scratch) + settings_path: Path | None = None + if agent == "codex": + if not CODEX_BINARY.is_file(): + raise EvaluationError(f"Codex binary is unavailable at {CODEX_BINARY}") + _prepare_codex_environment(external_run_root, scratch, environment) + command = _codex_command(workspace, scratch) + prompt_stdin = prompt + else: + if not CLAUDE_BINARY.is_file(): + raise EvaluationError(f"Claude binary is unavailable at {CLAUDE_BINARY}") + settings_path = run_root / "claude-settings.json" + _write_json(settings_path, _claude_settings(workspace, scratch)) + command = _claude_command(settings_path) + prompt_stdin = prompt + + cli_version = _binary_version(CODEX_BINARY if agent == "codex" else CLAUDE_BINARY, environment) + started_at = _utc_now() + metadata: dict[str, Any] = { + "protocol": PROTOCOL_ID, + "run_id": run_id, + "condition": condition, + "case_id": case_id, + "agent": agent, + "repetition": repetition, + "started_at": started_at, + "timeout_seconds": timeout_seconds, + "manifest_sha256": _sha256_file(manifest_path), + "gold_manifest_sha256": str(case["gold_sha256"]), + "runner_sha256": _sha256_file(RUNNER_PATH), + "protocol_sha256": _sha256_file(PROTOCOL_PATH), + "prompt_template_sha256": _sha256_file(PROMPT_PATH), + "rendered_prompt_sha256": _sha256_bytes(prompt.encode()), + "snapshot_sha256": _snapshot_digest(source), + "command": command, + "prompt_transport": "stdin", + "cli_version": cli_version, + "requested_model": "gpt-5.6-sol" if agent == "codex" else "opus", + "removed_environment_names": sorted(SECRET_ENVIRONMENT_NAMES), + "fixtures": fixtures, + } + _write_json(run_root / "metadata.json", metadata) + + exit_code, timed_out, elapsed_seconds = _run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt_stdin, + stdout_path=transcript_path, + stderr_path=stderr_path, + timeout_seconds=timeout_seconds, + ) + _write_workspace_evidence(workspace, run_root) + _record_final_fixture_state(fixtures, scratch) + shutil.copytree( + workspace, + run_root / "workspace-final", + symlinks=True, + ignore=shutil.ignore_patterns( + ".git", "bin", "obj", "target", "node_modules", ".eval-cache" + ), + ) + + metadata.update( + { + "completed_at": _utc_now(), + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed_seconds, + "transcript_sha256": _sha256_file(transcript_path), + "stderr_sha256": _sha256_file(stderr_path), + "fixtures": fixtures, + } + ) + model_identifier, model_identifier_source = _reported_model(transcript_path, agent) + metadata["model_identifier"] = model_identifier + metadata["model_identifier_source"] = model_identifier_source + _write_json(run_root / "metadata.json", metadata) + shutil.rmtree(external_run_root) + return metadata + + +def _parse_conditions(raw: str) -> list[str]: + conditions = [item.strip() for item in raw.split(",") if item.strip()] + unknown = [item for item in conditions if item not in {"baseline", "docs", "skill"}] + if unknown: + raise EvaluationError(f"unknown conditions: {', '.join(unknown)}") + if not conditions: + raise EvaluationError("at least one condition is required") + return conditions + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser("validate", help="validate frozen inputs") + validate.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH) + + plan = subparsers.add_parser("plan", help="create a deterministic run order") + plan.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH) + plan.add_argument("--output", type=Path, default=DEFAULT_PLAN_PATH) + plan.add_argument("--repetitions", type=int, default=DEFAULT_REPETITIONS) + plan.add_argument("--conditions", default="baseline") + + run = subparsers.add_parser("run", help="execute one isolated run") + run.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST_PATH) + run.add_argument("--case", required=True) + run.add_argument("--agent", choices=("codex", "claude"), required=True) + run.add_argument("--repetition", type=int, required=True) + run.add_argument("--condition", choices=("baseline", "docs", "skill"), default="baseline") + run.add_argument("--runs-path", type=Path, default=DEFAULT_RUNS_PATH) + run.add_argument("--timeout-seconds", type=int, default=DEFAULT_TIMEOUT_SECONDS) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + arguments = parser.parse_args(argv) + try: + if arguments.command == "validate": + manifest = validate_manifest(arguments.manifest) + print( + json.dumps( + { + "protocol": manifest["protocol"], + "cases": sorted(_case_index(manifest)), + "status": "valid", + }, + sort_keys=True, + ) + ) + elif arguments.command == "plan": + plan = create_plan( + arguments.manifest, + arguments.output, + arguments.repetitions, + _parse_conditions(arguments.conditions), + ) + print( + json.dumps( + { + "entries": len(plan["entries"]), + "output": str(arguments.output), + "status": "created", + }, + sort_keys=True, + ) + ) + elif arguments.command == "run": + metadata = execute_run( + arguments.manifest, + case_id=arguments.case, + agent=arguments.agent, + repetition=arguments.repetition, + condition=arguments.condition, + runs_path=arguments.runs_path, + timeout_seconds=arguments.timeout_seconds, + ) + print(json.dumps(metadata, sort_keys=True)) + else: # pragma: no cover - argparse enforces the command set + parser.error(f"unsupported command {arguments.command!r}") + except (EvaluationError, OSError, subprocess.CalledProcessError) as exc: + print(f"agent qualification evaluation failed: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/agent_qualification_generalization.py b/experiments/agent_qualification_generalization.py new file mode 100644 index 0000000..9390453 --- /dev/null +++ b/experiments/agent_qualification_generalization.py @@ -0,0 +1,541 @@ +"""Freeze and validate the v2 held-out agent-qualification cohort. + +This module intentionally does not launch provider CLIs. Execution remains +blocked until the user explicitly authorizes transmission of the new public +candidate snapshots and frozen treatment material to each provider. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import stat +from pathlib import Path +from typing import Any, Mapping, Sequence + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +PROTOCOL_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-generalization-v2.md" +PROMPT_PATH = REPOSITORY_ROOT / "experiments" / "agent_qualification_generalization_prompt.txt" +PRIVATE_ROOT = REPOSITORY_ROOT / "experiments" / "private" / "agent-qualification-v2" +DRAFT_PATH = PRIVATE_ROOT / "draft.json" +MANIFEST_PATH = PRIVATE_ROOT / "cases.json" +PLAN_PATH = PRIVATE_ROOT / "plan.json" +FREEZE_PATH = PRIVATE_ROOT / "freeze.json" +MANIFESTS_ROOT = PRIVATE_ROOT / "manifests" +SKILL_PATH = ( + PRIVATE_ROOT + / "treatments" + / "tracebook-qualify-matching-engine" + / "SKILL.md" +) + +PROTOCOL_ID = "agent-qualification-generalization-v2" +RANDOMIZATION_SEED = 20260730 +REPETITIONS = 3 +AGENTS = ("codex", "claude") +CONDITIONS = ("docs", "skill") + +DOCS_TREATMENT = ( + "\n\nYou may evaluate tracebook-conformance==0.6.0 using only its public " + "README and conformance documentation. Use it only if it fits the " + "declared contract.\n" +) + + +class GeneralizationError(RuntimeError): + """Raised when a frozen v2 precondition is not satisfied.""" + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _snapshot_digest(root: Path) -> str: + """Hash relative file names, modes, links, and bytes deterministically.""" + + digest = hashlib.sha256() + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative_path = path.relative_to(root) + if ".git" in relative_path.parts: + continue + relative = relative_path.as_posix() + if path.is_symlink(): + digest.update(b"L\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(os.readlink(path).encode()) + digest.update(b"\0") + elif path.is_file(): + digest.update(b"F\0") + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(f"{path.stat().st_mode & 0o777:o}".encode()) + digest.update(b"\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + digest.update(b"\0") + return digest.hexdigest() + + +def _tree_inventory(root: Path) -> list[dict[str, Any]]: + inventory: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise GeneralizationError(f"frozen tree must not contain symlink {relative!r}") + mode = path.stat().st_mode + if stat.S_ISDIR(mode): + continue + if not stat.S_ISREG(mode): + raise GeneralizationError(f"frozen tree contains unsafe entry {relative!r}") + inventory.append( + { + "path": relative, + "bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + ) + return inventory + + +def _load_json(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise GeneralizationError(f"cannot read JSON from {path}: {exc}") from exc + if not isinstance(payload, dict): + raise GeneralizationError(f"{path} must contain one JSON object") + return payload + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def _relative(path: Path) -> str: + return path.resolve().relative_to(REPOSITORY_ROOT.resolve()).as_posix() + + +def _private_path(raw_path: str, description: str) -> Path: + path = (REPOSITORY_ROOT / raw_path).resolve() + try: + path.relative_to(PRIVATE_ROOT.resolve()) + except ValueError as exc: + raise GeneralizationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc + return path + + +def _cases(payload: Mapping[str, Any]) -> list[dict[str, Any]]: + raw_cases = payload.get("cases") + if not isinstance(raw_cases, list) or not raw_cases: + raise GeneralizationError("case payload must contain a non-empty cases list") + result: list[dict[str, Any]] = [] + seen: set[str] = set() + for raw in raw_cases: + if not isinstance(raw, dict): + raise GeneralizationError("every case must be an object") + case_id = raw.get("id") + if not isinstance(case_id, str) or not case_id: + raise GeneralizationError("every case must have a non-empty id") + if case_id in seen: + raise GeneralizationError(f"duplicate case id {case_id!r}") + seen.add(case_id) + result.append(dict(raw)) + return result + + +def _freeze_tree( + *, + case_id: str, + name: str, + source_path: str, + snapshot_sha256: str, +) -> dict[str, Any]: + source = _private_path(source_path, f"{case_id} {name} source") + if not source.is_dir(): + raise GeneralizationError(f"{case_id} {name} source is unavailable: {source}") + inventory = _tree_inventory(source) + tree_sha256 = _snapshot_digest(source) + manifest = { + "kind": "frozen-tree-v1", + "case_id": case_id, + "name": name, + "snapshot_sha256": snapshot_sha256, + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": sum(int(item["bytes"]) for item in inventory), + "files": inventory, + } + manifest_path = MANIFESTS_ROOT / f"{case_id}--{name}.json" + _write_json(manifest_path, manifest) + return { + "kind": "frozen-tree-v1", + "source": _relative(source), + "manifest": _relative(manifest_path), + "manifest_sha256": _sha256_file(manifest_path), + "tree_sha256": tree_sha256, + } + + +def _validate_tree( + case: Mapping[str, Any], + declaration: Any, + *, + name: str, +) -> dict[str, Any]: + if not isinstance(declaration, dict) or declaration.get("kind") != "frozen-tree-v1": + raise GeneralizationError(f"case {case['id']!r} {name!r} must be frozen-tree-v1") + for field in ("source", "manifest", "manifest_sha256", "tree_sha256"): + if not isinstance(declaration.get(field), str): + raise GeneralizationError(f"case {case['id']!r} {name!r} lacks {field!r}") + source = _private_path(str(declaration["source"]), f"{case['id']} {name} source") + manifest_path = _private_path( + str(declaration["manifest"]), f"{case['id']} {name} manifest" + ) + if not source.is_dir() or not manifest_path.is_file(): + raise GeneralizationError(f"case {case['id']!r} {name!r} tree is unavailable") + manifest_hash = _sha256_file(manifest_path) + if manifest_hash != declaration["manifest_sha256"]: + raise GeneralizationError(f"case {case['id']!r} {name!r} manifest drifted") + manifest = _load_json(manifest_path) + inventory = _tree_inventory(source) + tree_sha256 = _snapshot_digest(source) + expected = { + "kind": "frozen-tree-v1", + "case_id": case["id"], + "name": name, + "snapshot_sha256": case["snapshot_sha256"], + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": sum(int(item["bytes"]) for item in inventory), + "files": inventory, + } + if manifest != expected: + raise GeneralizationError(f"case {case['id']!r} {name!r} inventory drifted") + if declaration["tree_sha256"] != tree_sha256: + raise GeneralizationError(f"case {case['id']!r} {name!r} tree drifted") + return { + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": expected["total_bytes"], + } + + +def freeze() -> dict[str, Any]: + """Create the immutable v2 case manifest, plan, and freeze binding.""" + + existing = [path for path in (MANIFEST_PATH, PLAN_PATH, FREEZE_PATH) if path.exists()] + if existing: + raise GeneralizationError( + "refusing to overwrite frozen v2 files: " + ", ".join(str(path) for path in existing) + ) + draft = _load_json(DRAFT_PATH) + if draft.get("protocol") != PROTOCOL_ID: + raise GeneralizationError(f"draft protocol must be {PROTOCOL_ID!r}") + if not PROTOCOL_PATH.is_file() or not PROMPT_PATH.is_file() or not SKILL_PATH.is_file(): + raise GeneralizationError("tracked protocol, prompt, and v2 skill must exist before freeze") + + frozen_cases: list[dict[str, Any]] = [] + for raw_case in _cases(draft): + required = ( + "source", + "revision", + "commands", + "declared_claim", + "excluded_scope", + "origin", + "gold_manifest", + "dependency_source", + "gold_evidence", + "status", + ) + missing = [field for field in required if field not in raw_case] + if missing: + raise GeneralizationError( + f"case {raw_case['id']!r} is missing fields: {', '.join(missing)}" + ) + source = _private_path(str(raw_case["source"]), f"{raw_case['id']} source") + if not source.is_dir() or (source / ".git").exists(): + raise GeneralizationError( + f"case {raw_case['id']!r} source must be an origin-stripped directory" + ) + snapshot_sha256 = _snapshot_digest(source) + gold_path = _private_path( + str(raw_case["gold_manifest"]), f"{raw_case['id']} gold manifest" + ) + if not gold_path.is_file(): + raise GeneralizationError(f"case {raw_case['id']!r} gold is unavailable") + evidence_specs = raw_case["gold_evidence"] + if not isinstance(evidence_specs, list): + raise GeneralizationError(f"case {raw_case['id']!r} gold_evidence must be a list") + evidence: list[dict[str, Any]] = [] + for item in evidence_specs: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise GeneralizationError("gold evidence entries require name and source") + evidence.append( + { + "name": item["name"], + **_freeze_tree( + case_id=str(raw_case["id"]), + name=str(item["name"]), + source_path=str(item.get("source", "")), + snapshot_sha256=snapshot_sha256, + ), + } + ) + dependency = _freeze_tree( + case_id=str(raw_case["id"]), + name="dependency-cache", + source_path=str(raw_case["dependency_source"]), + snapshot_sha256=snapshot_sha256, + ) + frozen_cases.append( + { + "id": raw_case["id"], + "source": _relative(source), + "snapshot_id": f"sha256:{snapshot_sha256}", + "snapshot_sha256": snapshot_sha256, + "revision": raw_case["revision"], + "commands": raw_case["commands"], + "declared_claim": raw_case["declared_claim"], + "excluded_scope": raw_case["excluded_scope"], + "origin": raw_case["origin"], + "gold_manifest": _relative(gold_path), + "gold_sha256": _sha256_file(gold_path), + "dependency_cache": dependency, + "gold_evidence": evidence, + "status": raw_case["status"], + } + ) + + manifest = { + "protocol": PROTOCOL_ID, + "protocol_sha256": _sha256_file(PROTOCOL_PATH), + "prompt_sha256": _sha256_file(PROMPT_PATH), + "skill_sha256": _sha256_file(SKILL_PATH), + "authorization": { + "codex": "pending-explicit-case-specific-authorization", + "claude": "pending-explicit-case-specific-authorization", + }, + "cases": frozen_cases, + } + _write_json(MANIFEST_PATH, manifest) + + plan = create_plan(manifest) + _write_json(PLAN_PATH, plan) + freeze_payload = { + "protocol": PROTOCOL_ID, + "randomization_seed": RANDOMIZATION_SEED, + "repetitions": REPETITIONS, + "agents": list(AGENTS), + "conditions": list(CONDITIONS), + "bindings": { + "draft": {"path": _relative(DRAFT_PATH), "sha256": _sha256_file(DRAFT_PATH)}, + "manifest": { + "path": _relative(MANIFEST_PATH), + "sha256": _sha256_file(MANIFEST_PATH), + }, + "plan": {"path": _relative(PLAN_PATH), "sha256": _sha256_file(PLAN_PATH)}, + "protocol": { + "path": _relative(PROTOCOL_PATH), + "sha256": _sha256_file(PROTOCOL_PATH), + }, + "prompt": {"path": _relative(PROMPT_PATH), "sha256": _sha256_file(PROMPT_PATH)}, + "runner": {"path": _relative(RUNNER_PATH), "sha256": _sha256_file(RUNNER_PATH)}, + "skill": {"path": _relative(SKILL_PATH), "sha256": _sha256_file(SKILL_PATH)}, + }, + "execution_authorized": False, + "execution_blocker": ( + "Explicit provider authorization for the new C4/C5 public snapshots " + "and frozen treatment material has not been recorded." + ), + } + _write_json(FREEZE_PATH, freeze_payload) + validate() + return freeze_payload + + +def validate_manifest() -> dict[str, Any]: + manifest = _load_json(MANIFEST_PATH) + if manifest.get("protocol") != PROTOCOL_ID: + raise GeneralizationError(f"manifest protocol must be {PROTOCOL_ID!r}") + tracked = { + "protocol_sha256": PROTOCOL_PATH, + "prompt_sha256": PROMPT_PATH, + "skill_sha256": SKILL_PATH, + } + for field, path in tracked.items(): + if manifest.get(field) != _sha256_file(path): + raise GeneralizationError(f"{field} drifted after freeze") + for case in _cases(manifest): + for field in ( + "source", + "snapshot_id", + "snapshot_sha256", + "revision", + "commands", + "declared_claim", + "excluded_scope", + "origin", + "gold_manifest", + "gold_sha256", + "dependency_cache", + "gold_evidence", + "status", + ): + if field not in case: + raise GeneralizationError(f"case {case['id']!r} is missing {field!r}") + source = _private_path(str(case["source"]), f"{case['id']} source") + if not source.is_dir() or (source / ".git").exists(): + raise GeneralizationError(f"case {case['id']!r} source is unavailable or not stripped") + snapshot_sha256 = _snapshot_digest(source) + if snapshot_sha256 != case["snapshot_sha256"]: + raise GeneralizationError(f"case {case['id']!r} snapshot drifted") + if case["snapshot_id"] != f"sha256:{snapshot_sha256}": + raise GeneralizationError(f"case {case['id']!r} snapshot_id is inconsistent") + gold_path = _private_path(str(case["gold_manifest"]), f"{case['id']} gold") + if _sha256_file(gold_path) != case["gold_sha256"]: + raise GeneralizationError(f"case {case['id']!r} gold drifted") + _validate_tree(case, case["dependency_cache"], name="dependency-cache") + evidence = case["gold_evidence"] + if not isinstance(evidence, list): + raise GeneralizationError(f"case {case['id']!r} gold_evidence must be a list") + names: set[str] = set() + for item in evidence: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise GeneralizationError("gold evidence entries require a name") + if item["name"] in names: + raise GeneralizationError(f"duplicate gold evidence name {item['name']!r}") + names.add(item["name"]) + _validate_tree(case, item, name=item["name"]) + return manifest + + +def create_plan(manifest: Mapping[str, Any]) -> dict[str, Any]: + ready = [case for case in _cases(manifest) if case.get("status") == "ready"] + entries: list[dict[str, Any]] = [] + for case in ready: + for agent in AGENTS: + for condition in CONDITIONS: + for repetition in range(1, REPETITIONS + 1): + entries.append( + { + "id": f"{condition}__{case['id']}__{agent}__r{repetition}", + "case_id": case["id"], + "agent": agent, + "condition": condition, + "repetition": repetition, + } + ) + random.Random(RANDOMIZATION_SEED).shuffle(entries) + for index, entry in enumerate(entries, 1): + entry["order"] = index + return { + "protocol": PROTOCOL_ID, + "randomization_seed": RANDOMIZATION_SEED, + "repetitions": REPETITIONS, + "agents": list(AGENTS), + "conditions": list(CONDITIONS), + "entry_count": len(entries), + "entries": entries, + } + + +def validate_plan(manifest: Mapping[str, Any]) -> dict[str, Any]: + plan = _load_json(PLAN_PATH) + expected = create_plan(manifest) + if plan != expected: + raise GeneralizationError("seeded v2 plan drifted") + return plan + + +def validate() -> dict[str, Any]: + manifest = validate_manifest() + plan = validate_plan(manifest) + freeze_payload = _load_json(FREEZE_PATH) + if freeze_payload.get("protocol") != PROTOCOL_ID: + raise GeneralizationError(f"freeze protocol must be {PROTOCOL_ID!r}") + bindings = freeze_payload.get("bindings") + if not isinstance(bindings, dict): + raise GeneralizationError("freeze bindings must be an object") + expected_paths = { + "draft": DRAFT_PATH, + "manifest": MANIFEST_PATH, + "plan": PLAN_PATH, + "protocol": PROTOCOL_PATH, + "prompt": PROMPT_PATH, + "runner": RUNNER_PATH, + "skill": SKILL_PATH, + } + for name, path in expected_paths.items(): + binding = bindings.get(name) + if not isinstance(binding, dict): + raise GeneralizationError(f"freeze lacks {name!r} binding") + if binding.get("path") != _relative(path) or binding.get("sha256") != _sha256_file(path): + raise GeneralizationError(f"freeze {name!r} binding drifted") + if freeze_payload.get("execution_authorized") is not False: + raise GeneralizationError("frozen cohort must remain execution-blocked") + return { + "protocol": PROTOCOL_ID, + "cases": len(_cases(manifest)), + "runs": plan["entry_count"], + "execution_authorized": False, + } + + +def render_prompt(case: Mapping[str, Any], condition: str) -> str: + if condition not in CONDITIONS: + raise GeneralizationError(f"condition must be one of {CONDITIONS!r}") + prompt = PROMPT_PATH.read_text() + substitutions = { + "case_id": case["id"], + "snapshot_id": case["snapshot_id"], + "revision": case["revision"], + "commands": case["commands"], + "declared_claim": case["declared_claim"], + "excluded_scope": case["excluded_scope"], + } + for name, value in substitutions.items(): + prompt = prompt.replace("{{" + name + "}}", str(value)) + if "{{" in prompt or "}}" in prompt: + raise GeneralizationError("unresolved prompt placeholder") + return prompt + DOCS_TREATMENT + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("freeze", "validate", "summary")) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "freeze": + payload = freeze() + elif args.command == "validate": + payload = validate() + else: + payload = validate() + payload["authorization_blocker"] = _load_json(FREEZE_PATH).get( + "execution_blocker" + ) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + except GeneralizationError as exc: + print(f"Generalization experiment error: {exc}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/agent_qualification_generalization_prompt.txt b/experiments/agent_qualification_generalization_prompt.txt new file mode 100644 index 0000000..c0f6b44 --- /dev/null +++ b/experiments/agent_qualification_generalization_prompt.txt @@ -0,0 +1,53 @@ +You are the release-validation engineer for a small venue-infrastructure team +that owns the matching logic in this repository. + +CASE: {{case_id}} +REPOSITORY SNAPSHOT: {{snapshot_id}} +PINNED REVISION: {{revision}} +SUPPORTED BUILD/TEST COMMANDS: {{commands}} +OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}} +EXCLUDED SCOPE: {{excluded_scope}} + +Determine whether this pinned repository can produce safe release evidence for +the declared matching semantics. Do not optimize for finding a bug. A correct +result may be conformant behavior, a candidate defect, an adapter/protocol +defect, a profile mismatch, insufficient observability, or an ambiguity that +requires maintainer adjudication. + +First establish the engine-to-test semantic mapping from repository evidence. +Do not silently invent lifecycle behavior, queue order, numeric conversions, +identities, matching responsibility, or unsupported features. Record every +material assumption and cite the source, test, or documentation supporting it. + +You may search for public development tools and their documentation, but do not +search for this candidate repository, revision, issues, known failures, prior +adapters, or experiment answers. Do not change production matching behavior. +Evaluation-only adapters, probes, artifacts, and native tests are allowed. + +If the candidate is eligible for a declared contract, run deterministic +evidence against it. If a disagreement appears, first rule out adapter, +protocol, normalization, observation, and profile errors. Attribute a +candidate defect only when intended behavior is independently established. + +Every terminal class requires two mechanically fresh evidence runs. Each run +must start from a fresh candidate copy, fresh build/cache target, fresh process, +and absent-before-use output path. Preserve both run identities and compare +their deterministic results. Do not overwrite an incomplete bundle or call a +second command over retained state a clean reproduction. + +Deliver: + +1. `CLASSIFICATION.md`: terminal class, scope, evidence, first disagreement + when applicable, and remaining uncertainty. +2. `MAPPING.md`: matching responsibility, event, numeric, identity, trade, + rejection, and ordered-state mappings with citations. +3. `COMMANDS.log`: every material command, working directory, target, and exit + code. +4. Applicable unmodified machine-readable qualification/failure artifacts. +5. `REPRODUCTION.md`: both fresh run identities, preconditions, outputs, and + equality checks. +6. A native regression only when candidate-defect intent is established. +7. A concise maintainer handoff that makes no claim beyond the evidence. + +Do not commit, push, open an issue, or contact a maintainer. Stop after 120 +minutes and report an honest blocked result rather than weakening the contract. diff --git a/experiments/agent_qualification_generalization_runs.py b/experiments/agent_qualification_generalization_runs.py new file mode 100644 index 0000000..ade1d11 --- /dev/null +++ b/experiments/agent_qualification_generalization_runs.py @@ -0,0 +1,1046 @@ +"""Execute the separately frozen v2 agent-qualification run matrix. + +The primary v2 cohort is immutable. This companion freezes machine/provider +bindings, verifies native skill delivery on synthetic material, enforces +provider-specific authorization, and records technical evidence without +grading semantic outcomes. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import socket +import subprocess +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +if __package__: + from . import agent_qualification as helpers + from . import agent_qualification_generalization as cohort + from . import agent_qualification_treatments as delivery +else: # Support direct execution from the repository root. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import agent_qualification as helpers # type: ignore[no-redef] + import agent_qualification_generalization as cohort # type: ignore[no-redef] + import agent_qualification_treatments as delivery # type: ignore[no-redef] + +ExecutionError = helpers.EvaluationError + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +TEST_PATH = REPOSITORY_ROOT / "tests" / "test_agent_qualification_generalization_runs.py" +ADDENDUM_PATH = REPOSITORY_ROOT / "docs" / "agent-qualification-generalization-execution-v2.md" +PRIVATE_ROOT = cohort.PRIVATE_ROOT +EXECUTION_ROOT = PRIVATE_ROOT / "execution" +RUNS_ROOT = EXECUTION_ROOT / "runs" +AUTHORIZATION_ROOT = EXECUTION_ROOT / "authorizations" +FREEZE_PATH = EXECUTION_ROOT / "freeze.json" +SHAKEDOWN_PATH = EXECUTION_ROOT / "native-delivery-shakedown.json" +CLAUDE_WRAPPER_PATH = ( + EXECUTION_ROOT / "provider-wrappers" / "claude" / ".claude-plugin" / "plugin.json" +) +EXTERNAL_ROOT = Path( + os.environ.get("TRACEBOOK_V2_EXTERNAL_ROOT", "/private/tmp/tracebook-agent-qualification-v2") +) + +PROTOCOL_ID = cohort.PROTOCOL_ID +FREEZE_KIND = "agent-qualification-generalization-execution-freeze-v1" +SHAKEDOWN_KIND = "agent-qualification-generalization-native-skill-shakedown-v1" +VERDICT_KIND = "agent-qualification-generalization-pre-score-verdict-v1" +AUTHORIZATION_KIND = "agent-qualification-generalization-provider-authorization-v1" +SKILL_NAME = "tracebook-qualify-matching-engine" +SYNTHETIC_SKILL_NAME = "native-delivery-probe" +SYNTHETIC_MARKER = "NATIVE_SKILL_PROBE_V2_20260801_7F31C9" +DEFAULT_TIMEOUT_SECONDS = 120 * 60 +EXPECTED_MODELS = {"codex": "gpt-5.6-sol", "claude": "claude-opus-4-8"} +PROVIDERS = tuple(EXPECTED_MODELS) +CLAUDE_FIRST_PARTY_ENV = { + "CLAUDE_CODE_USE_BEDROCK": "0", + "CLAUDE_CODE_USE_VERTEX": "0", + "CLAUDE_CODE_USE_FOUNDRY": "0", +} + +_PLUGIN_MANIFEST = { + "author": {"name": "Tracebook evaluator"}, + "description": "Isolated carrier for a preregistered agent evaluation treatment.", + "name": "agent-qualification-treatment", + "version": "2.0.0", +} +_SYNTHETIC_SKILL = ( + "---\n" + "name: native-delivery-probe\n" + "description: Synthetic probe used only to verify native skill delivery.\n" + "---\n\n" + "When explicitly asked to run this probe, reply with exactly " + SYNTHETIC_MARKER + ".\n" +).encode() + +_ORIGINAL_CODEX_SHELL_ENVIRONMENT = helpers._codex_shell_environment + + +def _json_bytes(value: Mapping[str, Any]) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _read_json(path: Path, description: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise ExecutionError(f"cannot read {description} from {path}: {exc}") from exc + if not isinstance(value, dict): + raise ExecutionError(f"{description} must contain one JSON object") + return value + + +def _write_new_json(path: Path, value: Mapping[str, Any], description: str) -> None: + if path.exists() or path.is_symlink(): + raise ExecutionError(f"refusing to overwrite {description} {path}") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(_json_bytes(value)) + + +def _file_record(path: Path) -> dict[str, Any]: + resolved = path.resolve(strict=True) + if not resolved.is_file(): + raise ExecutionError(f"required file is unavailable: {path}") + try: + label = resolved.relative_to(REPOSITORY_ROOT.resolve()).as_posix() + except ValueError: + label = str(resolved) + return { + "path": label, + "sha256": helpers._sha256_file(resolved), + "bytes": resolved.stat().st_size, + } + + +def _validate_file_record(record: Any, description: str) -> Path: + if not isinstance(record, dict): + raise ExecutionError(f"{description} binding is missing") + raw_path = record.get("path") + if not isinstance(raw_path, str): + raise ExecutionError(f"{description} path is missing") + candidate = Path(raw_path) + path = candidate if candidate.is_absolute() else REPOSITORY_ROOT / candidate + if _file_record(path) != record: + raise ExecutionError(f"{description} binding drifted") + return path.resolve() + + +def _provider_binary(provider: str) -> Path: + if provider not in PROVIDERS: + raise ExecutionError(f"unknown provider {provider!r}") + override = os.environ.get(f"TRACEBOOK_V2_{provider.upper()}_BINARY") + candidates = [ + Path(override) if override else None, + Path(shutil.which(provider) or "") if shutil.which(provider) else None, + Path.home() / ".local" / "bin" / provider, + Path("/Applications/ChatGPT.app/Contents/Resources/codex") if provider == "codex" else None, + ] + for candidate in candidates: + if candidate is not None and candidate.is_file(): + return candidate.resolve() + raise ExecutionError(f"{provider} CLI is unavailable") + + +def _clean_probe_environment() -> dict[str, str]: + secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + return { + name: value + for name, value in os.environ.items() + if name not in helpers.SECRET_ENVIRONMENT_NAMES + and not any(fragment in name.upper() for fragment in secret_fragments) + } + + +def _provider_binding(provider: str) -> dict[str, Any]: + binary = _provider_binary(provider) + version = helpers._binary_version(binary, _clean_probe_environment()) + binding: dict[str, Any] = { + "configured_path": str(binary), + "resolved_path": str(binary.resolve()), + "sha256": helpers._sha256_file(binary), + "bytes": binary.stat().st_size, + "cli_version": version, + "requested_model": EXPECTED_MODELS[provider], + } + if provider == "claude": + route_settings = json.dumps( + {"env": CLAUDE_FIRST_PARTY_ENV}, + sort_keys=True, + separators=(",", ":"), + ) + completed = subprocess.run( + ( + str(binary), + "--setting-sources", + "", + "--settings", + route_settings, + "auth", + "status", + ), + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=30, + ) + try: + auth = json.loads(completed.stdout) + except json.JSONDecodeError: + auth = {"raw_sha256": hashlib.sha256(completed.stdout.encode()).hexdigest()} + if isinstance(auth, dict): + # Do not persist account identifiers in the experiment archive. + binding["auth_route"] = { + key: auth.get(key) + for key in ("loggedIn", "authMethod", "apiProvider", "subscriptionType") + if key in auth + } + return binding + + +def _validated_provider_bindings() -> dict[str, dict[str, Any]]: + bindings = {provider: _provider_binding(provider) for provider in PROVIDERS} + route = bindings["claude"].get("auth_route") + if ( + not isinstance(route, dict) + or route.get("loggedIn") is not True + or route.get("apiProvider") != "firstParty" + ): + raise ExecutionError( + "Claude must use an authenticated first-party route before native-delivery " + "shakedown or execution freeze" + ) + return bindings + + +def _configure_delivery() -> None: + helpers.CODEX_BINARY = _provider_binary("codex") + helpers.CLAUDE_BINARY = _provider_binary("claude") + helpers.CODEX_AUTH_PATH = Path.home() / ".codex" / "auth.json" + delivery.DEFAULT_CLAUDE_WRAPPER_PATH = CLAUDE_WRAPPER_PATH + delivery.EXPECTED_MODELS = dict(EXPECTED_MODELS) + helpers._codex_shell_environment = _codex_shell_environment + + +def _toolchain_environment() -> dict[str, str]: + values: dict[str, str] = {} + path_prefixes: list[str] = [] + for name in ("TRACEBOOK_V2_MAVEN_HOME", "TRACEBOOK_V2_JAVA_HOME"): + raw = os.environ.get(name) + if not raw: + continue + root = Path(raw).resolve() + if not root.is_dir(): + raise ExecutionError(f"{name} is not an available directory: {root}") + path_prefixes.append(str(root / "bin")) + if name == "TRACEBOOK_V2_JAVA_HOME": + values["JAVA_HOME"] = str(root) + python_home = os.environ.get("TRACEBOOK_V2_PYTHON_HOME") + if python_home: + root = Path(python_home).resolve() + if not root.is_dir(): + raise ExecutionError(f"TRACEBOOK_V2_PYTHON_HOME is not an available directory: {root}") + path_prefixes.append(str(root / "bin")) + uv_binary = os.environ.get("TRACEBOOK_V2_UV_BINARY") + if uv_binary: + path = Path(uv_binary).resolve() + if not path.is_file(): + raise ExecutionError(f"TRACEBOOK_V2_UV_BINARY is not an available file: {path}") + path_prefixes.append(str(path.parent)) + cargo_bin = Path.home() / ".cargo" / "bin" + if cargo_bin.is_dir(): + path_prefixes.append(str(cargo_bin)) + values["PATH"] = ":".join( + [*path_prefixes, os.environ.get("PATH", "/usr/bin:/bin:/usr/sbin:/sbin")] + ) + return values + + +def _codex_shell_environment(scratch: Path) -> dict[str, str]: + values = _ORIGINAL_CODEX_SHELL_ENVIRONMENT(scratch) + values.update(_toolchain_environment()) + values.update(_fresh_environment(scratch)) + return values + + +def _claude_settings(workspace: Path, scratch: Path, plugin_root: Path) -> dict[str, Any]: + settings = delivery._claude_treatment_settings(workspace, scratch, plugin_root) + settings["env"].update(CLAUDE_FIRST_PARTY_ENV) + settings["env"].update(_toolchain_environment()) + settings["env"].update(_fresh_environment(scratch)) + return settings + + +def _fresh_environment(scratch: Path) -> dict[str, str]: + return { + "FRESH_M2": str(scratch / "m2-repository"), + "FRESH_CARGO_HOME": str(scratch / "cargo-home"), + "FRESH_TARGET": str(scratch / "cargo-target"), + } + + +def _write_wrapper() -> None: + if CLAUDE_WRAPPER_PATH.exists(): + if _read_json(CLAUDE_WRAPPER_PATH, "Claude plugin wrapper") != _PLUGIN_MANIFEST: + raise ExecutionError("Claude plugin wrapper drifted") + return + CLAUDE_WRAPPER_PATH.parent.mkdir(parents=True, exist_ok=True) + CLAUDE_WRAPPER_PATH.write_bytes(_json_bytes(_PLUGIN_MANIFEST)) + + +def _case_index(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + cases = manifest.get("cases") + if not isinstance(cases, list): + raise ExecutionError("validated cohort lost its cases") + return { + str(case["id"]): dict(case) + for case in cases + if isinstance(case, dict) and isinstance(case.get("id"), str) + } + + +def _plan_entry(plan: Mapping[str, Any], run_id: str) -> dict[str, Any]: + entries = plan.get("entries") + if not isinstance(entries, list): + raise ExecutionError("validated cohort lost its plan entries") + matches = [entry for entry in entries if isinstance(entry, dict) and entry.get("id") == run_id] + if len(matches) != 1: + raise ExecutionError(f"run {run_id!r} is not one frozen plan entry") + return dict(matches[0]) + + +def _load_primary() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + cohort.validate() + return ( + cohort._load_json(cohort.MANIFEST_PATH), + cohort._load_json(cohort.PLAN_PATH), + cohort._load_json(cohort.FREEZE_PATH), + ) + + +def _authorization_path(provider: str) -> Path: + if provider not in PROVIDERS: + raise ExecutionError(f"unknown provider {provider!r}") + return AUTHORIZATION_ROOT / f"{provider}.json" + + +def record_authorization(provider: str, statement_file: Path) -> dict[str, Any]: + """Record explicit consent already supplied by the user; never infer it.""" + + validate_freeze() + statement = statement_file.read_text().strip() + if not statement: + raise ExecutionError("authorization statement must not be empty") + payload = { + "kind": AUTHORIZATION_KIND, + "protocol": PROTOCOL_ID, + "provider": provider, + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + "user_statement": statement, + "statement_sha256": hashlib.sha256(statement.encode()).hexdigest(), + "recorded_at": helpers._utc_now(), + } + _write_new_json(_authorization_path(provider), payload, f"{provider} authorization") + return payload + + +def validate_authorization(provider: str) -> dict[str, Any]: + payload = _read_json(_authorization_path(provider), f"{provider} authorization") + expected = { + "kind": AUTHORIZATION_KIND, + "protocol": PROTOCOL_ID, + "provider": provider, + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + } + for field, value in expected.items(): + if payload.get(field) != value: + raise ExecutionError(f"{provider} authorization {field!r} is invalid") + statement = payload.get("user_statement") + if not isinstance(statement, str) or not statement.strip(): + raise ExecutionError(f"{provider} authorization statement is missing") + if payload.get("statement_sha256") != hashlib.sha256(statement.encode()).hexdigest(): + raise ExecutionError(f"{provider} authorization statement binding drifted") + if not isinstance(payload.get("recorded_at"), str): + raise ExecutionError(f"{provider} authorization timestamp is missing") + return payload + + +def _toolchain_versions() -> dict[str, Any]: + result: dict[str, Any] = {} + environment = dict(os.environ) + environment.update(_toolchain_environment()) + for name, command in { + "git": ("git", "--version"), + "python": (sys.executable, "--version"), + "java": ("java", "-version"), + "maven": ("mvn", "--version"), + "cargo": ("cargo", "--version"), + "rustc": ("rustc", "--version"), + }.items(): + binary = ( + shutil.which(command[0], path=environment["PATH"]) + if not Path(command[0]).is_absolute() + else command[0] + ) + if not binary: + result[name] = {"available": False} + continue + completed = subprocess.run( + (str(binary), *command[1:]), + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=30, + env=environment, + ) + result[name] = { + "available": completed.returncode == 0, + "path": str(Path(binary).resolve()), + "version": completed.stdout.strip().splitlines()[0] if completed.stdout.strip() else "", + } + return result + + +def _native_surface( + *, + provider: str, + condition: str, + workspace: Path, + run_root: Path, + scratch: Path, + environment: dict[str, str], + skill: bytes, + skill_name: str, +) -> tuple[list[str], dict[str, Any], Path | None]: + _configure_delivery() + return delivery._prepare_native_surface( + agent=provider, + condition=condition, + workspace=workspace, + external_run_root=run_root, + scratch=scratch, + environment=environment, + skill_content=skill, + skill_name=skill_name, + ) + + +def _claude_command(settings_path: Path, plugin_root: Path) -> list[str]: + command = delivery._claude_treatment_command(settings_path, plugin_root) + return command + + +def _run_delivery_probe(provider: str, condition: str, root: Path) -> dict[str, Any]: + workspace = root / "workspace" + scratch = root / "scratch" + workspace.mkdir(parents=True) + scratch.mkdir(parents=True) + (workspace / "README.md").write_text("Synthetic native skill delivery probe.\n") + environment = helpers._clean_agent_environment(scratch) + environment.update(_toolchain_environment()) + command, surface, plugin_root = _native_surface( + provider=provider, + condition=condition, + workspace=workspace, + run_root=root, + scratch=scratch, + environment=environment, + skill=_SYNTHETIC_SKILL, + skill_name=SYNTHETIC_SKILL_NAME, + ) + if provider == "claude": + if plugin_root is None: + raise ExecutionError("Claude probe plugin was not prepared") + settings_path = root / "claude-settings.json" + helpers._write_json(settings_path, _claude_settings(workspace, scratch, plugin_root)) + command = _claude_command(settings_path, plugin_root) + transcript = root / f"{provider}.jsonl" + stderr = root / f"{provider}.stderr" + prompt = ( + "Run the native-delivery-probe shakedown. If its native skill is available, " + "use it. If unavailable, reply exactly NO_NATIVE_DELIVERY. Do not search for " + "skill files or inspect any path not supplied by the native skill catalog." + ) + exit_code, timed_out, elapsed = helpers._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript, + stderr_path=stderr, + timeout_seconds=600, + ) + audit = delivery._audit_native_surface( + agent=provider, + condition=condition, + external_run_root=root, + scratch=scratch, + skill_content=_SYNTHETIC_SKILL, + skill_name=SYNTHETIC_SKILL_NAME, + ) + terminal = delivery._transcript_terminal_state(transcript, provider) + result = delivery._transcript_result_text(transcript, provider) + expected = SYNTHETIC_MARKER if condition == "skill" else "NO_NATIVE_DELIVERY" + errors = [] + if exit_code != 0 or timed_out or terminal != "completed": + errors.append("provider did not complete cleanly") + if audit.get("mutated") is not False: + errors.append("native skill surface mutated") + if result != expected: + errors.append(f"result {result!r} != {expected!r}") + return { + "valid": not errors, + "errors": errors, + "condition": condition, + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed, + "terminal_state": terminal, + "result": result, + "surface": audit, + "transcript": _file_record(transcript), + "stderr": _file_record(stderr), + } + + +def shakedown() -> dict[str, Any]: + cohort.validate() + _write_wrapper() + provider_bindings = _validated_provider_bindings() + if SHAKEDOWN_PATH.exists() or SHAKEDOWN_PATH.is_symlink(): + raise ExecutionError(f"refusing to overwrite shakedown {SHAKEDOWN_PATH}") + root = EXECUTION_ROOT / "shakedown-work" + if root.exists() or root.is_symlink(): + raise ExecutionError(f"refusing to reuse shakedown work root {root}") + providers: dict[str, Any] = {} + for provider in PROVIDERS: + conditions: dict[str, Any] = {} + for condition in ("docs", "skill"): + result = _run_delivery_probe(provider, condition, root / provider / condition) + if result["valid"] is not True: + raise ExecutionError( + f"{provider} {condition} native-delivery probe failed: " + + "; ".join(result["errors"]) + ) + conditions[condition] = result + providers[provider] = {"passed": True, "conditions": conditions} + verdict = { + "kind": SHAKEDOWN_KIND, + "protocol": PROTOCOL_ID, + "created_at": helpers._utc_now(), + "passed": True, + "runner": _file_record(RUNNER_PATH), + "synthetic_skill_sha256": hashlib.sha256(_SYNTHETIC_SKILL).hexdigest(), + "marker": SYNTHETIC_MARKER, + "provider_bindings": provider_bindings, + "providers": providers, + } + _write_new_json(SHAKEDOWN_PATH, verdict, "native-delivery shakedown") + return verdict + + +def _validate_shakedown() -> dict[str, Any]: + verdict = _read_json(SHAKEDOWN_PATH, "native-delivery shakedown") + if verdict.get("kind") != SHAKEDOWN_KIND or verdict.get("protocol") != PROTOCOL_ID: + raise ExecutionError("native-delivery shakedown identity changed") + if verdict.get("passed") is not True or verdict.get("runner") != _file_record(RUNNER_PATH): + raise ExecutionError("native-delivery shakedown binding changed") + if verdict.get("synthetic_skill_sha256") != hashlib.sha256(_SYNTHETIC_SKILL).hexdigest(): + raise ExecutionError("native-delivery synthetic skill changed") + if verdict.get("marker") != SYNTHETIC_MARKER: + raise ExecutionError("native-delivery marker changed") + if verdict.get("provider_bindings") != { + provider: _provider_binding(provider) for provider in PROVIDERS + }: + raise ExecutionError("native-delivery provider bindings changed") + providers = verdict.get("providers") + if not isinstance(providers, dict) or set(providers) != set(PROVIDERS): + raise ExecutionError("native-delivery provider coverage changed") + for provider in PROVIDERS: + conditions = ( + providers[provider].get("conditions") if isinstance(providers[provider], dict) else None + ) + if not isinstance(conditions, dict) or set(conditions) != {"docs", "skill"}: + raise ExecutionError(f"{provider} shakedown conditions changed") + if any( + result.get("valid") is not True + for result in conditions.values() + if isinstance(result, dict) + ): + raise ExecutionError(f"{provider} shakedown did not pass") + for result in conditions.values(): + if not isinstance(result, dict): + raise ExecutionError(f"{provider} shakedown result is invalid") + _validate_file_record(result.get("transcript"), f"{provider} transcript") + _validate_file_record(result.get("stderr"), f"{provider} stderr") + return verdict + + +def _freeze_payload(created_at: str) -> dict[str, Any]: + manifest, plan, primary_freeze = _load_primary() + del manifest, plan, primary_freeze + shakedown_verdict = _validate_shakedown() + inputs = { + "execution_addendum": _file_record(ADDENDUM_PATH), + "execution_runner": _file_record(RUNNER_PATH), + "execution_runner_tests": _file_record(TEST_PATH), + "helper_runner": _file_record(helpers.RUNNER_PATH), + "delivery_runner": _file_record(delivery.RUNNER_PATH), + "primary_protocol": _file_record(cohort.PROTOCOL_PATH), + "primary_prompt": _file_record(cohort.PROMPT_PATH), + "primary_runner": _file_record(cohort.RUNNER_PATH), + "primary_manifest": _file_record(cohort.MANIFEST_PATH), + "primary_plan": _file_record(cohort.PLAN_PATH), + "primary_freeze": _file_record(cohort.FREEZE_PATH), + "skill": _file_record(cohort.SKILL_PATH), + "claude_plugin_wrapper": _file_record(CLAUDE_WRAPPER_PATH), + "native_delivery_shakedown": _file_record(SHAKEDOWN_PATH), + } + return { + "kind": FREEZE_KIND, + "protocol": PROTOCOL_ID, + "created_at": created_at, + "host": { + "hostname": socket.gethostname(), + "platform": platform.platform(), + "machine": platform.machine(), + }, + "inputs": inputs, + "provider_bindings": shakedown_verdict["provider_bindings"], + "toolchains": _toolchain_versions(), + "run_count": 24, + "order": "exact seeded primary plan order", + "authorization": "provider-specific external records required after freeze", + "restart_policy": ( + "archive and retry only genuine technical interruptions without a completed " + "provider turn; never retry a timeout, refusal, or semantic outcome" + ), + } + + +def freeze_execution() -> dict[str, Any]: + if FREEZE_PATH.exists() or FREEZE_PATH.is_symlink(): + raise ExecutionError(f"refusing to overwrite execution freeze {FREEZE_PATH}") + payload = _freeze_payload(helpers._utc_now()) + _write_new_json(FREEZE_PATH, payload, "execution freeze") + return payload + + +def validate_freeze() -> dict[str, Any]: + cohort.validate() + payload = _read_json(FREEZE_PATH, "execution freeze") + if payload.get("kind") != FREEZE_KIND or payload.get("protocol") != PROTOCOL_ID: + raise ExecutionError("execution freeze identity changed") + created_at = payload.get("created_at") + if not isinstance(created_at, str) or not created_at: + raise ExecutionError("execution freeze timestamp is missing") + if payload != _freeze_payload(created_at): + raise ExecutionError("execution freeze content changed") + return payload + + +def _copy_fixture(case: Mapping[str, Any], scratch: Path) -> dict[str, Any]: + declaration = case.get("dependency_cache") + if not isinstance(declaration, dict): + raise ExecutionError("case dependency-cache declaration is missing") + source = (REPOSITORY_ROOT / str(declaration["source"])).resolve(strict=True) + target_name = "m2-repository" if case["id"] == "c4-compatible" else "cargo-home" + target = scratch / target_name + helpers._copy_frozen_tree(source, target, str(declaration["tree_sha256"])) + return { + "source_manifest_sha256": declaration["manifest_sha256"], + "initial_tree_sha256": declaration["tree_sha256"], + "target": str(target), + } + + +def _record_fixture_final(fixture: dict[str, Any]) -> None: + target = Path(str(fixture["target"])) + if not target.is_dir(): + fixture["final_state"] = "missing" + return + fixture["final_state"] = "present" + fixture["final_tree_sha256"] = helpers._snapshot_digest(target) + + +def _run_root(run_id: str) -> Path: + return RUNS_ROOT / run_id + + +def _external_run_root(run_id: str) -> Path: + return EXTERNAL_ROOT / run_id + + +def _technical_verdict( + run_root: Path, metadata: Mapping[str, Any], errors: list[str] +) -> dict[str, Any]: + verdict = { + "kind": VERDICT_KIND, + "protocol": PROTOCOL_ID, + "run_id": metadata.get("run_id"), + "valid": not errors, + "errors": errors, + "terminal_state": metadata.get("terminal_state"), + "timed_out": metadata.get("timed_out"), + "exit_code": metadata.get("exit_code"), + "metadata_sha256": helpers._sha256_file(run_root / "metadata.json"), + "transcript_sha256": metadata.get("transcript_sha256"), + "created_at": helpers._utc_now(), + } + _write_new_json(run_root / "pre-score-verdict.json", verdict, "pre-score verdict") + return verdict + + +def validate_run(run_id: str) -> dict[str, Any]: + freeze = validate_freeze() + manifest, plan, _ = _load_primary() + entry = _plan_entry(plan, run_id) + run_root = _run_root(run_id) + metadata = _read_json(run_root / "metadata.json", "run metadata") + verdict = _read_json(run_root / "pre-score-verdict.json", "pre-score verdict") + errors: list[str] = [] + expected = { + "protocol": PROTOCOL_ID, + "run_id": run_id, + "case_id": entry["case_id"], + "provider": entry["agent"], + "condition": entry["condition"], + "repetition": entry["repetition"], + "order": entry["order"], + "primary_manifest_sha256": helpers._sha256_file(cohort.MANIFEST_PATH), + "primary_plan_sha256": helpers._sha256_file(cohort.PLAN_PATH), + "primary_freeze_sha256": helpers._sha256_file(cohort.FREEZE_PATH), + "execution_freeze_sha256": helpers._sha256_file(FREEZE_PATH), + "execution_runner_sha256": helpers._sha256_file(RUNNER_PATH), + } + for field, value in expected.items(): + if metadata.get(field) != value: + errors.append(f"metadata {field} binding changed") + case = _case_index(manifest)[str(entry["case_id"])] + if metadata.get("snapshot_sha256") != case["snapshot_sha256"]: + errors.append("candidate snapshot binding changed") + if metadata.get("initial_workspace_sha256") != case["snapshot_sha256"]: + errors.append("initial workspace binding changed") + if metadata.get("frozen_skill_sha256") != helpers._sha256_file(cohort.SKILL_PATH): + errors.append("frozen skill metadata binding changed") + transcript = run_root / f"{entry['agent']}.jsonl" + stderr = run_root / f"{entry['agent']}.stderr" + prompt = run_root / "prompt.txt" + for artifact in (transcript, stderr, prompt): + if not artifact.is_file() or artifact.is_symlink(): + errors.append(f"missing required artifact {artifact.name}") + if transcript.is_file() and metadata.get("transcript_sha256") != helpers._sha256_file( + transcript + ): + errors.append("transcript hash changed") + if stderr.is_file() and metadata.get("stderr_sha256") != helpers._sha256_file(stderr): + errors.append("stderr hash changed") + rendered = cohort.render_prompt(case, str(entry["condition"])) + rendered_hash = hashlib.sha256(rendered.encode()).hexdigest() + if metadata.get("rendered_prompt_sha256") != rendered_hash: + errors.append("rendered prompt metadata binding changed") + if prompt.is_file() and helpers._sha256_file(prompt) != rendered_hash: + errors.append("rendered prompt artifact changed") + frozen_provider = freeze["provider_bindings"][entry["agent"]] + if metadata.get("provider_binding") != frozen_provider: + errors.append("provider binding changed") + if metadata.get("authorization_sha256") != helpers._sha256_file( + _authorization_path(str(entry["agent"])) + ): + errors.append("provider authorization binding changed") + if metadata.get("gold_manifest_sha256") != case["gold_sha256"]: + errors.append("gold manifest binding changed") + if metadata.get("model_identifier") != EXPECTED_MODELS[str(entry["agent"])]: + errors.append("reported model identifier changed") + if metadata.get("exit_code") != 0: + errors.append("provider subprocess exit code changed") + if metadata.get("terminal_state") != "completed" or metadata.get("timed_out") is not False: + errors.append("provider did not complete a valid technical turn") + surface = metadata.get("native_surface") + expected_skill = entry["condition"] == "skill" + if not isinstance(surface, dict) or surface.get("skill_installed") is not expected_skill: + errors.append("native skill surface condition changed") + else: + expected_skill_hash = helpers._sha256_file(cohort.SKILL_PATH) if expected_skill else None + if surface.get("semantic_file_count") != (1 if expected_skill else 0): + errors.append("native skill semantic file count changed") + if surface.get("skill_sha256") != expected_skill_hash: + errors.append("native skill hash changed") + if ( + not isinstance(surface.get("final_audit"), dict) + or surface["final_audit"].get("mutated") is not False + ): + errors.append("native skill surface audit failed") + for artifact_name in ( + "git-status.txt", + "workspace.patch", + "workspace-files.json", + "workspace-final", + ): + if not (run_root / artifact_name).exists(): + errors.append(f"missing required artifact {artifact_name}") + if verdict.get("kind") != VERDICT_KIND or verdict.get("run_id") != run_id: + errors.append("pre-score verdict identity changed") + if verdict.get("metadata_sha256") != helpers._sha256_file(run_root / "metadata.json"): + errors.append("pre-score verdict metadata binding changed") + if verdict.get("transcript_sha256") != metadata.get("transcript_sha256"): + errors.append("pre-score verdict transcript binding changed") + if verdict.get("valid") is not True or verdict.get("errors") != []: + errors.append("pre-score verdict is not valid") + if errors: + raise ExecutionError("; ".join(errors)) + return verdict + + +def _validate_prior_runs(plan: Mapping[str, Any], order: int) -> None: + entries = plan.get("entries") + if not isinstance(entries, list): + raise ExecutionError("validated cohort lost its plan entries") + for prior in entries[: order - 1]: + if not isinstance(prior, dict): + raise ExecutionError("plan entry shape changed") + validate_run(str(prior["id"])) + + +def execute_run(run_id: str, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS) -> dict[str, Any]: + if timeout_seconds != DEFAULT_TIMEOUT_SECONDS: + raise ExecutionError(f"official timeout must remain {DEFAULT_TIMEOUT_SECONDS} seconds") + freeze = validate_freeze() + manifest, plan, _ = _load_primary() + entry = _plan_entry(plan, run_id) + provider = str(entry["agent"]) + validate_authorization(provider) + _validate_prior_runs(plan, int(entry["order"])) + case = _case_index(manifest)[str(entry["case_id"])] + run_root = _run_root(run_id) + external_root = _external_run_root(run_id) + if run_root.exists() or run_root.is_symlink(): + raise ExecutionError(f"refusing to overwrite existing run {run_root}") + if external_root.exists() or external_root.is_symlink(): + raise ExecutionError(f"refusing to reuse external workspace {external_root}") + + source = (REPOSITORY_ROOT / str(case["source"])).resolve(strict=True) + workspace = external_root / "workspace" + scratch = external_root / "scratch" + run_root.mkdir(parents=True) + scratch.mkdir(parents=True) + helpers._initialize_snapshot(source, workspace) + initial_workspace_hash = helpers._snapshot_digest(workspace) + if initial_workspace_hash != case["snapshot_sha256"]: + raise ExecutionError("fresh workspace changed the frozen candidate snapshot") + for relative in (".tracebook", ".eval-cache", "target"): + if (workspace / relative).exists() or (workspace / relative).is_symlink(): + raise ExecutionError(f"fresh workspace unexpectedly contains {relative}") + prompt = cohort.render_prompt(case, str(entry["condition"])) + (run_root / "prompt.txt").write_text(prompt) + environment = helpers._clean_agent_environment(scratch) + environment.update(_toolchain_environment()) + fixture = _copy_fixture(case, scratch) + environment.update(_fresh_environment(scratch)) + Path(environment["FRESH_TARGET"]).mkdir(exist_ok=True) + skill = cohort.SKILL_PATH.read_bytes() + command, surface, plugin_root = _native_surface( + provider=provider, + condition=str(entry["condition"]), + workspace=workspace, + run_root=external_root, + scratch=scratch, + environment=environment, + skill=skill, + skill_name=SKILL_NAME, + ) + if provider == "claude": + if plugin_root is None: + raise ExecutionError("Claude plugin was not prepared") + settings_path = run_root / "claude-settings.json" + helpers._write_json(settings_path, _claude_settings(workspace, scratch, plugin_root)) + command = _claude_command(settings_path, plugin_root) + + transcript = run_root / f"{provider}.jsonl" + stderr = run_root / f"{provider}.stderr" + binary = _provider_binary(provider) + metadata: dict[str, Any] = { + "protocol": PROTOCOL_ID, + "run_id": run_id, + "case_id": entry["case_id"], + "provider": provider, + "condition": entry["condition"], + "repetition": entry["repetition"], + "order": entry["order"], + "started_at": helpers._utc_now(), + "timeout_seconds": timeout_seconds, + "primary_manifest_sha256": helpers._sha256_file(cohort.MANIFEST_PATH), + "primary_plan_sha256": helpers._sha256_file(cohort.PLAN_PATH), + "primary_freeze_sha256": helpers._sha256_file(cohort.FREEZE_PATH), + "execution_freeze_sha256": helpers._sha256_file(FREEZE_PATH), + "execution_runner_sha256": helpers._sha256_file(RUNNER_PATH), + "snapshot_sha256": case["snapshot_sha256"], + "initial_workspace_sha256": initial_workspace_hash, + "gold_manifest_sha256": case["gold_sha256"], + "rendered_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "authorization_sha256": helpers._sha256_file(_authorization_path(provider)), + "provider_binding": freeze["provider_bindings"][provider], + "command": command, + "requested_model": EXPECTED_MODELS[provider], + "cli_version": helpers._binary_version(binary, environment), + "binary_sha256": helpers._sha256_file(binary), + "prompt_transport": "stdin", + "fixture": fixture, + "native_surface": surface, + "skill_injected": entry["condition"] == "skill", + "frozen_skill_sha256": helpers._sha256_file(cohort.SKILL_PATH), + } + helpers._write_json(run_root / "metadata.json", metadata) + exit_code, timed_out, elapsed = helpers._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript, + stderr_path=stderr, + timeout_seconds=timeout_seconds, + ) + surface["final_audit"] = delivery._audit_native_surface( + agent=provider, + condition=str(entry["condition"]), + external_run_root=external_root, + scratch=scratch, + skill_content=skill, + skill_name=SKILL_NAME, + ) + helpers._write_workspace_evidence(workspace, run_root) + _record_fixture_final(fixture) + shutil.copytree( + workspace, + run_root / "workspace-final", + symlinks=True, + ignore=shutil.ignore_patterns(".git", "target", "node_modules", ".eval-cache"), + ) + terminal = delivery._transcript_terminal_state(transcript, provider) + model, model_source = helpers._reported_model(transcript, provider) + metadata.update( + { + "completed_at": helpers._utc_now(), + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed, + "terminal_state": terminal, + "model_identifier": model, + "model_identifier_source": model_source, + "transcript_sha256": helpers._sha256_file(transcript), + "stderr_sha256": helpers._sha256_file(stderr), + "fixture": fixture, + "native_surface": surface, + } + ) + helpers._write_json(run_root / "metadata.json", metadata) + errors: list[str] = [] + if timed_out: + errors.append("provider subprocess timed out") + if terminal != "completed": + errors.append(f"provider terminal state was {terminal!r}") + if exit_code != 0: + errors.append(f"provider exit code was {exit_code}") + if model != EXPECTED_MODELS[provider]: + errors.append(f"reported model {model!r} != {EXPECTED_MODELS[provider]!r}") + if surface["final_audit"].get("mutated") is not False: + errors.append("native skill surface mutated") + verdict = _technical_verdict(run_root, metadata, errors) + if errors: + raise ExecutionError(f"run {run_id!r} is technically invalid: " + "; ".join(errors)) + validate_run(run_id) + shutil.rmtree(external_root) + return {"metadata": metadata, "verdict": verdict} + + +def status() -> dict[str, Any]: + cohort_summary = cohort.validate() + _, plan, _ = _load_primary() + completed = [] + for entry in plan["entries"]: + run_id = str(entry["id"]) + if (_run_root(run_id) / "pre-score-verdict.json").is_file(): + try: + validate_run(run_id) + except ExecutionError: + continue + completed.append(run_id) + authorizations = {provider: _authorization_path(provider).is_file() for provider in PROVIDERS} + next_run = next( + (entry["id"] for entry in plan["entries"] if entry["id"] not in completed), None + ) + return { + **cohort_summary, + "execution_frozen": FREEZE_PATH.is_file(), + "authorizations_recorded": authorizations, + "completed_valid_runs": len(completed), + "next_run": next_run, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("shakedown") + commands.add_parser("freeze") + commands.add_parser("validate") + commands.add_parser("status") + authorize = commands.add_parser("authorize") + authorize.add_argument("--provider", required=True, choices=PROVIDERS) + authorize.add_argument("--statement-file", required=True, type=Path) + run = commands.add_parser("run") + run.add_argument("--run-id", required=True) + run.add_argument("--timeout-seconds", type=int, default=DEFAULT_TIMEOUT_SECONDS) + validate_run_parser = commands.add_parser("validate-run") + validate_run_parser.add_argument("--run-id", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "shakedown": + payload = shakedown() + elif args.command == "freeze": + payload = freeze_execution() + elif args.command == "validate": + payload = validate_freeze() + elif args.command == "authorize": + payload = record_authorization(args.provider, args.statement_file) + elif args.command == "run": + payload = execute_run(args.run_id, args.timeout_seconds) + elif args.command == "validate-run": + payload = validate_run(args.run_id) + else: + payload = status() + except (ExecutionError, OSError, ValueError, subprocess.SubprocessError) as exc: + print(f"Execution experiment error: {exc}", file=sys.stderr) + return 2 + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/agent_qualification_prompt.txt b/experiments/agent_qualification_prompt.txt new file mode 100644 index 0000000..419565d --- /dev/null +++ b/experiments/agent_qualification_prompt.txt @@ -0,0 +1,51 @@ +You are the release-validation engineer for a small venue-infrastructure team +that owns the matching logic in this repository. + +CASE: {{case_id}} +REPOSITORY SNAPSHOT: {{snapshot_id}} +PINNED REVISION: {{revision}} +SUPPORTED BUILD/TEST COMMANDS: {{commands}} +OWNER'S ONLY DECLARED GUARANTEE: {{declared_claim}} +EXCLUDED SCOPE: {{excluded_scope}} + +Determine whether this pinned engine preserves the declared matching semantics. +Do not optimize for finding a bug. A correct result may be: + +- conformant behavior; +- a candidate-engine defect; +- an adapter or observation defect; +- a profile/contract mismatch; +- insufficient observability; or +- an ambiguity requiring maintainer adjudication. + +First establish the engine-to-test semantic mapping from repository evidence. +Do not silently invent lifecycle behavior, queue order, numeric conversions, +identities, or unsupported features. Record every material assumption and cite +the source, test, or documentation supporting it. + +Use any public development tool you can discover if it is appropriate. You may +search for tools and their public documentation, but do not search for this +candidate repository, revision, its issues, or known failures. Do not change +production matching behavior. Evaluation-only adapters, artifacts, and native +tests are allowed. + +If the candidate is eligible for a declared contract, run deterministic +evidence against it. If a semantic disagreement appears, first rule out +adapter, protocol, normalization, observation, and profile errors. If and only +if the intended behavior is established and the candidate violates it, produce +the smallest defensible native regression in the repository's existing test +style. + +Deliver: + +1. CLASSIFICATION.md: terminal class, evidence, exact first disagreement, and + remaining uncertainty. +2. MAPPING.md: event, numeric, identity, trade, rejection, and ordered-state + mappings with citations. +3. COMMANDS.log: every material command and exit code. +4. Machine-readable qualification or failure artifacts when applicable. +5. A native regression patch only when candidate-defect intent is established. +6. A concise maintainer handoff that makes no claim beyond the evidence. + +Do not commit, push, open an issue, or contact a maintainer. Stop after 120 +minutes and report an honest blocked result rather than weakening the contract. diff --git a/experiments/agent_qualification_treatments.py b/experiments/agent_qualification_treatments.py new file mode 100644 index 0000000..b023805 --- /dev/null +++ b/experiments/agent_qualification_treatments.py @@ -0,0 +1,1692 @@ +"""Frozen docs/skill treatment runner for the agent qualification evaluation. + +The baseline runner is a scored, hash-pinned input and must remain unchanged. +This companion imports its isolation and evidence helpers while adding only the +pre-registered docs and native-skill treatment surfaces. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import shlex +import shutil +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence +from urllib.parse import urlparse + +if __package__: + from . import agent_qualification as baseline +else: # Support ``python experiments/agent_qualification_treatments.py``. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import agent_qualification as baseline # type: ignore[no-redef] + +EvaluationError = baseline.EvaluationError + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = Path(__file__).resolve() +PRIVATE_ROOT = baseline.PRIVATE_ROOT +TREATMENT_ROOT = PRIVATE_ROOT / "treatments" +DEFAULT_SKILL_PATH = TREATMENT_ROOT / "tracebook-qualify-matching-engine" / "SKILL.md" +DEFAULT_EXTERNAL_PINS_PATH = TREATMENT_ROOT / "EXTERNAL_PINS.json" +DEFAULT_PROTOCOL_AMENDMENT_PATH = TREATMENT_ROOT / "TREATMENT_PROTOCOL.md" +DEFAULT_AUTHORIZATION_PATH = TREATMENT_ROOT / "AUTHORIZATION.json" +DEFAULT_CLAUDE_WRAPPER_PATH = ( + TREATMENT_ROOT / "provider-wrappers" / "claude" / ".claude-plugin" / "plugin.json" +) +DEFAULT_PLAN_PATH = TREATMENT_ROOT / "master-plan.json" +DEFAULT_FREEZE_PATH = TREATMENT_ROOT / "freeze.json" +DEFAULT_SHAKEDOWN_VERDICT_PATH = TREATMENT_ROOT / "native-delivery-shakedown.json" +DEFAULT_BASELINE_PLAN_PATH = baseline.DEFAULT_PLAN_PATH +EVALUATION_ROOT = PRIVATE_ROOT / "evaluation" +DEFAULT_RUBRIC_PATH = EVALUATION_ROOT / "rubric.json" +DEFAULT_VALIDATOR_PATH = EVALUATION_ROOT / "scorecard.py" +DEFAULT_BASELINE_SCORES_PATH = EVALUATION_ROOT / "scores" +DEFAULT_RUNS_PATH = baseline.DEFAULT_RUNS_PATH +DEFAULT_TEST_PATH = REPOSITORY_ROOT / "tests" / "test_agent_qualification_treatments.py" + +PROTOCOL_ID = baseline.PROTOCOL_ID +PLAN_KIND = "agent-qualification-treatment-master-plan-v1" +FREEZE_KIND = "agent-qualification-treatment-freeze-v1" +VERDICT_KIND = "agent-qualification-pre-score-run-verdict-v1" +SHAKEDOWN_KIND = "agent-qualification-native-skill-shakedown-v1" +TREATMENT_CONDITIONS = ("docs", "skill") +REPETITIONS = 3 +RUNS_PER_CONDITION = 12 +TOTAL_RUNS = 24 +RANDOMIZATION_SEED = baseline.RANDOMIZATION_SEED +SKILL_NAME = "tracebook-qualify-matching-engine" +EXPECTED_MODELS = { + "codex": "gpt-5.6-sol", + "claude": "claude-opus-4-8", +} + +MEASUREMENT_POLICY: dict[str, Any] = { + "assigned_payload_is_intervention": False, + "post_launch_evaluator_intervention": "I0", + "elapsed_endpoint": { + "name": "subprocess completion elapsed", + "definition": ( + "monotonic wall-clock seconds from provider subprocess spawn " "until termination" + ), + "not_measured": "time-to-first-valid-evidence", + }, + "timing_gate": { + "minimum_matched_autonomous_safe_pass_pairs": 3, + "both_cases_required": True, + "skill_safe_pass_rate_must_not_be_lower": True, + "paired_statistic": "median(skill_elapsed/docs_elapsed)", + "pass_threshold": 0.70, + "zero_c3_safety_violations_required": True, + "otherwise": ( + "timing is exploratory/inestimable; skill value may pass only " + "through at least +20 percentage points safe-pass rate" + ), + }, +} + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _json_bytes(value: Mapping[str, Any]) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _read_json_once(path: Path, description: str) -> tuple[dict[str, Any], bytes]: + try: + raw = path.read_bytes() + value = json.loads(raw) + except (OSError, json.JSONDecodeError) as exc: + raise EvaluationError(f"cannot read {description} from {path}: {exc}") from exc + if not isinstance(value, dict): + raise EvaluationError(f"{description} must contain one JSON object") + return value, raw + + +def _path_label(path: Path) -> str: + resolved = path.resolve() + try: + return resolved.relative_to(REPOSITORY_ROOT.resolve()).as_posix() + except ValueError: + return str(resolved) + + +def _reject_symlink_chain(path: Path, root: Path, description: str) -> None: + lexical_path = Path(os.path.abspath(path)) + lexical_root = Path(os.path.abspath(root)) + try: + relative = lexical_path.relative_to(lexical_root) + except ValueError as exc: + raise EvaluationError(f"{description} must stay inside {lexical_root}") from exc + cursor = lexical_root + if cursor.is_symlink(): + raise EvaluationError(f"{description} root must not be a symlink: {cursor}") + for part in relative.parts: + cursor /= part + if cursor.is_symlink(): + raise EvaluationError(f"{description} must not use symlinks: {cursor}") + + +def _private_file(path: Path, description: str) -> Path: + candidate = path if path.is_absolute() else REPOSITORY_ROOT / path + _reject_symlink_chain(candidate, PRIVATE_ROOT, description) + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(PRIVATE_ROOT.resolve(strict=True)) + except (OSError, ValueError) as exc: + raise EvaluationError(f"{description} must be an existing private file") from exc + mode = resolved.stat().st_mode + if not stat.S_ISREG(mode): + raise EvaluationError(f"{description} must be a regular file: {resolved}") + return resolved + + +def _private_output(path: Path, description: str) -> Path: + candidate = path if path.is_absolute() else REPOSITORY_ROOT / path + parent = candidate.parent + _reject_symlink_chain(parent, PRIVATE_ROOT, description) + parent.mkdir(parents=True, exist_ok=True) + try: + parent.resolve(strict=True).relative_to(PRIVATE_ROOT.resolve(strict=True)) + except (OSError, ValueError) as exc: + raise EvaluationError(f"{description} must stay inside {PRIVATE_ROOT}") from exc + if candidate.exists() or candidate.is_symlink(): + raise EvaluationError(f"refusing to overwrite {description}: {candidate}") + return candidate + + +def _file_record(path: Path) -> dict[str, Any]: + return { + "path": _path_label(path), + "sha256": baseline._sha256_file(path), + "bytes": path.stat().st_size, + } + + +def _tree_record(root: Path) -> dict[str, Any]: + resolved = root.resolve(strict=True) + inventory = baseline._tree_inventory(resolved) + return { + "path": _path_label(resolved), + "sha256": baseline._snapshot_digest(resolved), + "file_count": len(inventory), + "files": inventory, + } + + +def _resolve_record( + record: Any, + description: str, + *, + expected_path: Path | None = None, + private: bool, +) -> tuple[Path, bytes]: + if not isinstance(record, dict): + raise EvaluationError(f"{description} record must be an object") + raw_path = record.get("path") + expected_hash = record.get("sha256") + expected_bytes = record.get("bytes") + if not isinstance(raw_path, str) or not isinstance(expected_hash, str): + raise EvaluationError(f"{description} record must pin path and sha256") + candidate = Path(raw_path) + if private: + path = _private_file(candidate, description) + else: + lexical_path = candidate if candidate.is_absolute() else REPOSITORY_ROOT / candidate + _reject_symlink_chain(lexical_path, REPOSITORY_ROOT, description) + path = lexical_path.resolve(strict=True) + if not path.is_file(): + raise EvaluationError(f"{description} must be a regular non-symlink file") + if expected_path is not None and path != expected_path.resolve(strict=True): + raise EvaluationError(f"{description} path changed: expected {expected_path}, got {path}") + raw = path.read_bytes() + actual_hash = _sha256_bytes(raw) + if actual_hash != expected_hash: + raise EvaluationError(f"{description} changed: expected {expected_hash}, got {actual_hash}") + if expected_bytes != len(raw): + raise EvaluationError(f"{description} byte length changed") + return path, raw + + +def _write_new_json(path: Path, payload: Mapping[str, Any], description: str) -> None: + target = _private_output(path, description) + target.write_bytes(_json_bytes(payload)) + + +def _collect_strings(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, dict): + result: list[str] = [] + for nested in value.values(): + result.extend(_collect_strings(nested)) + return result + if isinstance(value, list): + result = [] + for nested in value: + result.extend(_collect_strings(nested)) + return result + return [] + + +def _leakage_tokens(manifest: Mapping[str, Any]) -> set[str]: + tokens: set[str] = set() + cases = manifest.get("cases") + if not isinstance(cases, list): + return tokens + for case in cases: + if not isinstance(case, dict): + continue + selected: list[Any] = [ + case.get("id"), + case.get("snapshot_id"), + case.get("snapshot_sha256"), + case.get("revision"), + case.get("source"), + case.get("gold_manifest"), + case.get("gold_sha256"), + case.get("origin"), + ] + for raw in _collect_strings(selected): + normalized = raw.strip().casefold() + if len(normalized) >= 8: + tokens.add(normalized) + if "://" in raw: + parsed = urlparse(raw) + for part in parsed.path.split("/"): + part = part.strip().casefold() + if len(part) >= 8: + tokens.add(part) + return tokens + + +def _load_skill(skill_path: Path, manifest: Mapping[str, Any]) -> dict[str, Any]: + path = _private_file(skill_path, "skill") + raw = path.read_bytes() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise EvaluationError("skill must be UTF-8") from exc + if text.encode("utf-8") != raw: + raise EvaluationError("skill bytes must round-trip through UTF-8 exactly") + if not text.startswith("---\n") or f"name: {SKILL_NAME}\n" not in text: + raise EvaluationError(f"skill frontmatter must name {SKILL_NAME!r}") + lowered = text.casefold() + leaked = sorted(token for token in _leakage_tokens(manifest) if token in lowered) + if leaked: + raise EvaluationError( + "skill leakage audit found held-out identifier(s): " + ", ".join(leaked) + ) + return { + **_file_record(path), + "name": SKILL_NAME, + "content": raw, + } + + +def _ready_case_ids(manifest: Mapping[str, Any]) -> list[str]: + return sorted( + case_id + for case_id, case in baseline._case_index(manifest).items() + if case.get("status") == "ready" + ) + + +def _structural_cells(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: + cells = [ + { + "case_id": case_id, + "agent": agent, + "repetition": repetition, + } + for case_id in _ready_case_ids(manifest) + for agent in ("codex", "claude") + for repetition in range(1, REPETITIONS + 1) + ] + random.Random(RANDOMIZATION_SEED).shuffle(cells) + if len(cells) != RUNS_PER_CONDITION: + raise EvaluationError( + f"treatment protocol requires {RUNS_PER_CONDITION} cells, got {len(cells)}" + ) + return cells + + +def _evidence_id(ordinal: int) -> str: + material = f"{PROTOCOL_ID}|treatment|{RANDOMIZATION_SEED}|{ordinal}".encode() + return "ev_" + _sha256_bytes(material)[:20] + + +def _master_entries(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: + cells = _structural_cells(manifest) + entries: list[dict[str, Any]] = [] + ordinal = 0 + for condition in TREATMENT_CONDITIONS: + for cell in cells: + ordinal += 1 + run_id = f"{condition}__{cell['case_id']}__{cell['agent']}__r{cell['repetition']}" + entries.append( + { + **cell, + "condition": condition, + "run_id": run_id, + "evidence_id": _evidence_id(ordinal), + "ordinal": ordinal, + } + ) + return entries + + +def render_prompt(case: Mapping[str, Any]) -> str: + """Render the byte-identical user prompt used by both treatment arms.""" + + return baseline.render_prompt(case, "docs") + + +def _binary_binding(agent: str) -> dict[str, Any]: + binary = baseline.CODEX_BINARY if agent == "codex" else baseline.CLAUDE_BINARY + if not binary.is_file(): + raise EvaluationError(f"{agent} binary is unavailable at {binary}") + secret_fragments = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + clean_environment = { + name: value + for name, value in os.environ.items() + if name not in baseline.SECRET_ENVIRONMENT_NAMES + and not any(fragment in name.upper() for fragment in secret_fragments) + } + version = baseline._binary_version(binary, clean_environment) + return { + "configured_path": str(binary), + "resolved_path": str(binary.resolve()), + "sha256": baseline._sha256_file(binary), + "bytes": binary.stat().st_size, + "cli_version": version, + "requested_model": EXPECTED_MODELS[agent], + "required_reported_model": EXPECTED_MODELS[agent], + } + + +def _load_external_pins(path: Path) -> tuple[dict[str, Any], dict[str, Any]]: + resolved = _private_file(path, "external pins") + payload, _ = _read_json_once(resolved, "external pins") + if payload.get("protocol") != PROTOCOL_ID: + raise EvaluationError("external pins protocol changed") + models = payload.get("models") + release = payload.get("public_release") + if not isinstance(models, dict) or not isinstance(release, dict): + raise EvaluationError("external pins must contain models and public_release") + for agent in ("codex", "claude"): + expected = _binary_binding(agent) + pinned = models.get(agent) + if not isinstance(pinned, dict): + raise EvaluationError(f"external pins are missing {agent}") + checks = { + "binary_path": expected["configured_path"], + "binary_sha256": expected["sha256"], + "cli_version": expected["cli_version"], + "requested_model": expected["requested_model"], + "required_reported_model": expected["required_reported_model"], + } + for field, expected_value in checks.items(): + if pinned.get(field) != expected_value: + raise EvaluationError( + f"external {agent} pin {field!r} changed: " + f"expected {expected_value!r}, got {pinned.get(field)!r}" + ) + git = release.get("git") + pypi = release.get("pypi") + if not isinstance(git, dict) or not isinstance(pypi, dict): + raise EvaluationError("external release pins must contain git and pypi") + required_release_values = { + ("git", "tag"): "v0.6.0", + ("git", "tag_commit"): "d7e20a345974f8fb57537512962e283f14ae51e3", + ("pypi", "project"): "tracebook-conformance", + ("pypi", "version"): "0.6.0", + } + sections = {"git": git, "pypi": pypi} + for (section, field), expected_value in required_release_values.items(): + if sections[section].get(field) != expected_value: + raise EvaluationError( + f"public release pin {section}.{field} must be {expected_value!r}" + ) + for section, fields in { + "git": ("tagged_readme_sha256", "tagged_conformance_docs_sha256"), + "pypi": ( + "wheel_filename", + "wheel_sha256", + "sdist_filename", + "sdist_sha256", + ), + }.items(): + for field in fields: + value = sections[section].get(field) + if not isinstance(value, str) or not value: + raise EvaluationError(f"public release pin {section}.{field} is missing") + return payload, _file_record(resolved) + + +def _prompt_hashes( + manifest: Mapping[str, Any], +) -> dict[str, dict[str, str]]: + result: dict[str, dict[str, str]] = {} + for case_id in _ready_case_ids(manifest): + prompt = render_prompt(baseline._case_index(manifest)[case_id]) + prompt_hash = _sha256_bytes(prompt.encode()) + result[case_id] = {"docs": prompt_hash, "skill": prompt_hash} + return result + + +def _plan_payload( + manifest_path: Path, + manifest: Mapping[str, Any], + skill: Mapping[str, Any], + external_pins: Mapping[str, Any], + external_pins_record: Mapping[str, Any], + protocol_amendment_path: Path, + authorization_path: Path, + claude_wrapper_path: Path, +) -> dict[str, Any]: + entries = _master_entries(manifest) + prompt_hashes = _prompt_hashes(manifest) + for entry in entries: + entry["rendered_prompt_sha256"] = prompt_hashes[entry["case_id"]][entry["condition"]] + return { + "kind": PLAN_KIND, + "protocol": PROTOCOL_ID, + "created_at": baseline._utc_now(), + "randomization_seed": RANDOMIZATION_SEED, + "design": "docs-block-then-matched-skill-block", + "repetitions": REPETITIONS, + "runs_per_condition": RUNS_PER_CONDITION, + "timeout_seconds": baseline.DEFAULT_TIMEOUT_SECONDS, + "conditions": list(TREATMENT_CONDITIONS), + "manifest": _file_record(manifest_path), + "baseline_runner": _file_record(baseline.RUNNER_PATH), + "treatment_runner": _file_record(RUNNER_PATH), + "protocol_document": _file_record(baseline.PROTOCOL_PATH), + "prompt_template": _file_record(baseline.PROMPT_PATH), + "treatment_protocol": _file_record(protocol_amendment_path), + "authorization": _file_record(authorization_path), + "docs_treatment_sha256": _sha256_bytes(baseline.DOCS_TREATMENT.encode()), + "skill": {key: value for key, value in skill.items() if key != "content"}, + "native_wrappers": { + "codex": { + "kind": "isolated-codex-home-skill-v1", + "semantic_files": ["SKILL.md"], + }, + "claude": { + "kind": "isolated-claude-plugin-skill-v1", + "manifest": _file_record(claude_wrapper_path), + "source_tree": _tree_record(claude_wrapper_path.parents[1]), + "semantic_files": ["SKILL.md"], + }, + }, + "external_pins": dict(external_pins_record), + "pinned_models": external_pins["models"], + "public_release": external_pins["public_release"], + "rendered_prompt_sha256": prompt_hashes, + "measurement_policy": MEASUREMENT_POLICY, + "entries": entries, + } + + +def create_plan( + manifest_path: Path, + *, + skill_path: Path, + external_pins_path: Path, + protocol_amendment_path: Path, + authorization_path: Path, + claude_wrapper_path: Path, + output_path: Path, +) -> dict[str, Any]: + manifest_path = manifest_path.resolve(strict=True) + manifest = baseline.validate_manifest(manifest_path) + skill = _load_skill(skill_path, manifest) + external_pins, external_pins_record = _load_external_pins(external_pins_path) + amendment = _private_file(protocol_amendment_path, "treatment protocol") + authorization = _private_file(authorization_path, "authorization ledger") + wrapper = _private_file(claude_wrapper_path, "Claude plugin wrapper") + payload = _plan_payload( + manifest_path, + manifest, + skill, + external_pins, + external_pins_record, + amendment, + authorization, + wrapper, + ) + _write_new_json(output_path, payload, "master plan") + return payload + + +def _validate_plan_entry_shape(entries: Any) -> list[dict[str, Any]]: + if not isinstance(entries, list): + raise EvaluationError("master plan entries must be a list") + if len(entries) != TOTAL_RUNS: + raise EvaluationError(f"master plan must contain {TOTAL_RUNS} entries") + result: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + seen_evidence: set[str] = set() + for index, entry in enumerate(entries, start=1): + if not isinstance(entry, dict): + raise EvaluationError("every master plan entry must be an object") + if entry.get("ordinal") != index: + raise EvaluationError("master plan ordinals must be contiguous") + run_id = entry.get("run_id") + evidence_id = entry.get("evidence_id") + if not isinstance(run_id, str) or run_id in seen_ids: + raise EvaluationError("master plan run IDs must be unique strings") + if not isinstance(evidence_id, str) or evidence_id in seen_evidence: + raise EvaluationError("master plan evidence IDs must be unique opaque strings") + if entry.get("condition") not in TREATMENT_CONDITIONS: + raise EvaluationError("master plan contains an invalid condition") + seen_ids.add(run_id) + seen_evidence.add(evidence_id) + result.append(entry) + return result + + +def validate_plan( + plan_path: Path, + *, + manifest_path: Path, + skill_path: Path, + external_pins_path: Path, + protocol_amendment_path: Path, + authorization_path: Path, + claude_wrapper_path: Path, +) -> dict[str, Any]: + path = _private_file(plan_path, "master plan") + payload, raw = _read_json_once(path, "master plan") + manifest_path = manifest_path.resolve(strict=True) + manifest = baseline.validate_manifest(manifest_path) + skill = _load_skill(skill_path, manifest) + external_pins, external_record = _load_external_pins(external_pins_path) + amendment = _private_file(protocol_amendment_path, "treatment protocol") + authorization = _private_file(authorization_path, "authorization ledger") + wrapper = _private_file(claude_wrapper_path, "Claude plugin wrapper") + + if payload.get("kind") != PLAN_KIND or payload.get("protocol") != PROTOCOL_ID: + raise EvaluationError("master plan identity changed") + if payload.get("randomization_seed") != RANDOMIZATION_SEED: + raise EvaluationError("master plan randomization seed changed") + if payload.get("conditions") != list(TREATMENT_CONDITIONS): + raise EvaluationError("master plan condition order changed") + if payload.get("repetitions") != REPETITIONS: + raise EvaluationError("master plan repetition count changed") + if payload.get("timeout_seconds") != baseline.DEFAULT_TIMEOUT_SECONDS: + raise EvaluationError("master plan timeout changed") + expected_records = { + "manifest": _file_record(manifest_path), + "baseline_runner": _file_record(baseline.RUNNER_PATH), + "treatment_runner": _file_record(RUNNER_PATH), + "protocol_document": _file_record(baseline.PROTOCOL_PATH), + "prompt_template": _file_record(baseline.PROMPT_PATH), + "treatment_protocol": _file_record(amendment), + "authorization": _file_record(authorization), + "external_pins": external_record, + } + for field, expected in expected_records.items(): + if payload.get(field) != expected: + raise EvaluationError(f"master plan {field} binding changed") + expected_skill = {key: value for key, value in skill.items() if key != "content"} + if payload.get("skill") != expected_skill: + raise EvaluationError("master plan skill binding changed") + if payload.get("docs_treatment_sha256") != _sha256_bytes(baseline.DOCS_TREATMENT.encode()): + raise EvaluationError("master plan docs treatment binding changed") + if payload.get("pinned_models") != external_pins["models"]: + raise EvaluationError("master plan model pins changed") + if payload.get("public_release") != external_pins["public_release"]: + raise EvaluationError("master plan public release pins changed") + if payload.get("measurement_policy") != MEASUREMENT_POLICY: + raise EvaluationError("master plan measurement policy changed") + native_wrappers = payload.get("native_wrappers") + if not isinstance(native_wrappers, dict): + raise EvaluationError("master plan native wrapper bindings are missing") + claude_wrapper = native_wrappers.get("claude") + if not isinstance(claude_wrapper, dict) or claude_wrapper.get("manifest") != _file_record( + wrapper + ): + raise EvaluationError("master plan Claude wrapper binding changed") + if claude_wrapper.get("source_tree") != _tree_record(wrapper.parents[1]): + raise EvaluationError("master plan Claude wrapper tree changed") + if payload.get("rendered_prompt_sha256") != _prompt_hashes(manifest): + raise EvaluationError("master plan rendered prompt hashes changed") + + entries = _validate_plan_entry_shape(payload.get("entries")) + expected_entries = _master_entries(manifest) + prompt_hashes = _prompt_hashes(manifest) + for entry in expected_entries: + entry["rendered_prompt_sha256"] = prompt_hashes[entry["case_id"]][entry["condition"]] + if entries != expected_entries: + raise EvaluationError("master plan entry order or identity changed") + payload["_path"] = str(path) + payload["_sha256"] = _sha256_bytes(raw) + payload["_skill_content"] = skill["content"] + return payload + + +def _run_scorecard_validator(*args: str) -> dict[str, Any] | None: + completed = subprocess.run( + (sys.executable, str(DEFAULT_VALIDATOR_PATH), *args), + cwd=REPOSITORY_ROOT, + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise EvaluationError(f"scorecard validation failed: {detail}") + if not completed.stdout.strip().startswith("{"): + return None + value = json.loads(completed.stdout) + if not isinstance(value, dict): + raise EvaluationError("scorecard validator returned a non-object") + return value + + +def _validate_baseline_evidence() -> dict[str, Any]: + _run_scorecard_validator("check-rubric") + plan_path = _private_file(DEFAULT_BASELINE_PLAN_PATH, "baseline plan") + plan, _ = _read_json_once(plan_path, "baseline plan") + manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH) + expected_entries = [ + { + "run_id": (f"baseline__{cell['case_id']}__{cell['agent']}__r" f"{cell['repetition']}"), + "condition": "baseline", + **cell, + } + for cell in _structural_cells(manifest) + ] + expected_plan_fields = { + "protocol": PROTOCOL_ID, + "randomization_seed": RANDOMIZATION_SEED, + "repetitions": REPETITIONS, + "conditions": ["baseline"], + "manifest_sha256": baseline._sha256_file(baseline.DEFAULT_MANIFEST_PATH), + "runner_sha256": baseline._sha256_file(baseline.RUNNER_PATH), + } + for field, expected in expected_plan_fields.items(): + if plan.get(field) != expected: + raise EvaluationError(f"baseline plan {field} binding changed") + if plan.get("entries") != expected_entries: + raise EvaluationError("baseline plan identity or order changed") + + paths = sorted(DEFAULT_BASELINE_SCORES_PATH.glob("baseline__*.scorecard.json")) + if len(paths) != RUNS_PER_CONDITION: + raise EvaluationError( + f"expected {RUNS_PER_CONDITION} baseline scorecards, got {len(paths)}" + ) + expected_run_ids = {entry["run_id"] for entry in expected_entries} + records: list[dict[str, Any]] = [] + observed_run_ids: set[str] = set() + for path in paths: + summary = _run_scorecard_validator("validate", str(path)) + scorecard, _ = _read_json_once(path, "baseline scorecard") + run_id = scorecard.get("run_id") + if not isinstance(run_id, str) or run_id in observed_run_ids: + raise EvaluationError("baseline scorecard run IDs must be unique") + if summary is None or summary.get("protocol_valid") is not True: + raise EvaluationError(f"baseline scorecard {run_id!r} is not protocol-valid") + observed_run_ids.add(run_id) + records.append( + { + "run_id": run_id, + "case_id": scorecard.get("case_id"), + "score_summary": summary, + **_file_record(path), + } + ) + if observed_run_ids != expected_run_ids: + raise EvaluationError("baseline scorecard identities do not match the baseline plan") + return { + "plan": _file_record(plan_path), + "rubric": _file_record(DEFAULT_RUBRIC_PATH), + "validator": _file_record(DEFAULT_VALIDATOR_PATH), + "scorecards": records, + } + + +def _validate_shakedown_verdict(path: Path) -> dict[str, Any]: + resolved = _private_file(path, "native-delivery shakedown verdict") + verdict, _ = _read_json_once(resolved, "native-delivery shakedown verdict") + if verdict.get("kind") != SHAKEDOWN_KIND or verdict.get("protocol") != PROTOCOL_ID: + raise EvaluationError("native-delivery shakedown identity changed") + if verdict.get("runner") != _file_record(RUNNER_PATH): + raise EvaluationError("native-delivery shakedown runner binding changed") + if verdict.get("passed") is not True: + raise EvaluationError("native-delivery shakedown did not pass") + providers = verdict.get("providers") + if not isinstance(providers, dict) or set(providers) != {"codex", "claude"}: + raise EvaluationError("native-delivery shakedown must cover Codex and Claude") + skill_hash = verdict.get("synthetic_skill_sha256") + marker = verdict.get("marker") + if not isinstance(skill_hash, str) or not isinstance(marker, str): + raise EvaluationError("native-delivery shakedown is missing its canary binding") + for agent, result in providers.items(): + if not isinstance(result, dict) or result.get("passed") is not True: + raise EvaluationError(f"native-delivery shakedown failed for {agent}") + if result.get("synthetic_skill_sha256") != skill_hash: + raise EvaluationError(f"native-delivery skill bytes differed for {agent}") + if result.get("treatment_result") != marker: + raise EvaluationError(f"native-delivery marker was not observed for {agent}") + if result.get("control_result") != "NO_NATIVE_DELIVERY": + raise EvaluationError(f"native-delivery control was contaminated for {agent}") + for field in ("control_transcript", "treatment_transcript"): + record = result.get(field) + _resolve_record(record, f"{agent} {field}", private=True) + return {"verdict": verdict, "record": _file_record(resolved)} + + +def _freeze_payload( + *, + created_at: str, + plan: Mapping[str, Any], + shakedown: Mapping[str, Any], +) -> dict[str, Any]: + authorization = _private_file(DEFAULT_AUTHORIZATION_PATH, "authorization ledger") + wrapper = _private_file(DEFAULT_CLAUDE_WRAPPER_PATH, "Claude plugin wrapper") + external_pins = _private_file(DEFAULT_EXTERNAL_PINS_PATH, "external pins") + treatment_protocol = _private_file(DEFAULT_PROTOCOL_AMENDMENT_PATH, "treatment protocol") + test_path = DEFAULT_TEST_PATH.resolve(strict=True) + return { + "kind": FREEZE_KIND, + "protocol": PROTOCOL_ID, + "created_at": created_at, + "plan": _file_record(Path(str(plan["_path"]))), + "baseline": _validate_baseline_evidence(), + "frozen_inputs": { + "baseline_runner": _file_record(baseline.RUNNER_PATH), + "treatment_runner": _file_record(RUNNER_PATH), + "treatment_runner_tests": _file_record(test_path), + "protocol_document": _file_record(baseline.PROTOCOL_PATH), + "prompt_template": _file_record(baseline.PROMPT_PATH), + "case_manifest": _file_record(baseline.DEFAULT_MANIFEST_PATH), + "treatment_protocol": _file_record(treatment_protocol), + "authorization": _file_record(authorization), + "external_pins": _file_record(external_pins), + "claude_plugin_wrapper": _file_record(wrapper), + "skill": dict(plan["skill"]), + }, + "binary_bindings": {agent: _binary_binding(agent) for agent in ("codex", "claude")}, + "native_delivery_shakedown": dict(shakedown["record"]), + "measurement_policy": MEASUREMENT_POLICY, + "conditions": list(TREATMENT_CONDITIONS), + "condition_order": "all docs verdicts, then all skill verdicts", + "post_launch_scoring": ( + "No measured outcome grading until all 24 pre-score verdicts validate" + ), + "restart_policy": ( + "Quarantine and clean-restart only genuine technical interruptions; " + "never retry a completed result, refusal, timeout, or semantic failure" + ), + } + + +def create_freeze( + freeze_path: Path, + *, + plan_path: Path, + shakedown_verdict_path: Path, +) -> dict[str, Any]: + plan = validate_plan( + plan_path, + manifest_path=baseline.DEFAULT_MANIFEST_PATH, + skill_path=DEFAULT_SKILL_PATH, + external_pins_path=DEFAULT_EXTERNAL_PINS_PATH, + protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH, + authorization_path=DEFAULT_AUTHORIZATION_PATH, + claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH, + ) + shakedown = _validate_shakedown_verdict(shakedown_verdict_path) + payload = _freeze_payload( + created_at=baseline._utc_now(), + plan=plan, + shakedown=shakedown, + ) + _write_new_json(freeze_path, payload, "treatment freeze") + return payload + + +def validate_freeze( + freeze_path: Path, + *, + plan_path: Path = DEFAULT_PLAN_PATH, + shakedown_verdict_path: Path = DEFAULT_SHAKEDOWN_VERDICT_PATH, +) -> dict[str, Any]: + path = _private_file(freeze_path, "treatment freeze") + payload, raw = _read_json_once(path, "treatment freeze") + if payload.get("kind") != FREEZE_KIND or payload.get("protocol") != PROTOCOL_ID: + raise EvaluationError("treatment freeze identity changed") + created_at = payload.get("created_at") + if not isinstance(created_at, str) or not created_at: + raise EvaluationError("treatment freeze is missing created_at") + plan = validate_plan( + plan_path, + manifest_path=baseline.DEFAULT_MANIFEST_PATH, + skill_path=DEFAULT_SKILL_PATH, + external_pins_path=DEFAULT_EXTERNAL_PINS_PATH, + protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH, + authorization_path=DEFAULT_AUTHORIZATION_PATH, + claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH, + ) + shakedown = _validate_shakedown_verdict(shakedown_verdict_path) + expected = _freeze_payload( + created_at=created_at, + plan=plan, + shakedown=shakedown, + ) + if payload != expected: + raise EvaluationError("treatment freeze content changed") + payload["_path"] = str(path) + payload["_sha256"] = _sha256_bytes(raw) + payload["_plan"] = plan + return payload + + +def _claude_treatment_settings( + workspace: Path, + scratch: Path, + plugin_root: Path, +) -> dict[str, Any]: + settings = baseline._claude_settings(workspace, scratch) + environment = settings["env"] + environment.pop("HOME", None) + zdotdir = scratch / "zsh-env" + zdotdir.mkdir() + zshenv = zdotdir / ".zshenv" + zshenv.write_text(f"export HOME={shlex.quote(str(scratch / 'shell-home'))}\n") + zshenv.chmod(0o444) + environment["ZDOTDIR"] = str(zdotdir) + environment["CLAUDE_CODE_SAFE_MODE"] = "0" + permissions = settings["permissions"] + plugin_rule = "//" + str(plugin_root).lstrip("/") + zshenv_rule = "//" + str(zshenv).lstrip("/") + permissions["deny"].append(f"Edit({plugin_rule}/**)") + permissions["deny"].append(f"Edit({zshenv_rule})") + sandbox = settings["sandbox"] + sandbox["filesystem"]["denyWrite"].append(str(plugin_root)) + sandbox["filesystem"]["denyWrite"].append(str(zshenv)) + return settings + + +def _claude_treatment_command( + settings_path: Path, + plugin_root: Path, +) -> list[str]: + return [ + str(baseline.CLAUDE_BINARY), + "-p", + "--no-session-persistence", + "--no-chrome", + "--setting-sources", + "", + "--strict-mcp-config", + "--mcp-config", + '{"mcpServers":{}}', + "--settings", + str(settings_path), + "--plugin-dir", + str(plugin_root), + "--permission-mode", + "dontAsk", + "--tools", + "Read,Glob,Grep,Edit,Write,Bash,WebSearch,WebFetch,Skill", + "--model", + EXPECTED_MODELS["claude"], + "--effort", + "high", + "--max-budget-usd", + "50", + "--output-format", + "stream-json", + "--verbose", + ] + + +def _codex_treatment_command( + workspace: Path, + scratch: Path, + skills_root: Path, +) -> list[str]: + command = baseline._codex_command(workspace, scratch) + permission_index = next( + index for index, value in enumerate(command) if value.startswith("permissions.agent-eval=") + ) + permission_profile = command[permission_index] + skill_read_entry = f'{json.dumps(str(skills_root))}="read",' + command[permission_index] = permission_profile.replace( + "filesystem={", + "filesystem={" + skill_read_entry, + 1, + ) + return command + + +def _prepare_native_surface( + *, + agent: str, + condition: str, + workspace: Path, + external_run_root: Path, + scratch: Path, + environment: dict[str, str], + skill_content: bytes, + skill_name: str = SKILL_NAME, +) -> tuple[list[str], dict[str, Any], Path | None]: + skill_hash = _sha256_bytes(skill_content) + if condition not in TREATMENT_CONDITIONS: + raise EvaluationError(f"unknown treatment condition {condition!r}") + installed = condition == "skill" + if agent == "codex": + codex_home = baseline._prepare_codex_environment(external_run_root, scratch, environment) + skills_root = codex_home / "skills" + skills_root.mkdir(exist_ok=True) + if installed: + target = skills_root / skill_name / "SKILL.md" + target.parent.mkdir() + target.write_bytes(skill_content) + target.chmod(0o444) + surface = { + "kind": "isolated-codex-home-skill-v1", + "skill_installed": installed, + "semantic_file_count": 1 if installed else 0, + "skill_sha256": skill_hash if installed else None, + } + return _codex_treatment_command(workspace, scratch, skills_root), surface, None + + if agent != "claude": + raise EvaluationError(f"unknown agent {agent!r}") + plugin_source = DEFAULT_CLAUDE_WRAPPER_PATH.parents[1] + plugin_root = scratch / "claude-plugin" + shutil.copytree(plugin_source, plugin_root) + if installed: + target = plugin_root / "skills" / skill_name / "SKILL.md" + target.parent.mkdir(parents=True) + target.write_bytes(skill_content) + target.chmod(0o444) + surface = { + "kind": "isolated-claude-plugin-skill-v1", + "skill_installed": installed, + "semantic_file_count": 1 if installed else 0, + "skill_sha256": skill_hash if installed else None, + "wrapper_manifest_sha256": baseline._sha256_file( + plugin_root / ".claude-plugin" / "plugin.json" + ), + } + return [], surface, plugin_root + + +def _audit_native_surface( + *, + agent: str, + condition: str, + external_run_root: Path, + scratch: Path, + skill_content: bytes, + skill_name: str = SKILL_NAME, +) -> dict[str, Any]: + if agent == "codex": + path = external_run_root / "codex-home" / "skills" / skill_name / "SKILL.md" + elif agent == "claude": + path = scratch / "claude-plugin" / "skills" / skill_name / "SKILL.md" + else: + raise EvaluationError(f"unknown agent {agent!r}") + expected = condition == "skill" + present = path.is_file() and not path.is_symlink() + actual_hash = baseline._sha256_file(path) if present else None + expected_hash = _sha256_bytes(skill_content) if expected else None + return { + "expected_present": expected, + "present": present, + "path": str(path), + "expected_sha256": expected_hash, + "final_sha256": actual_hash, + "mutated": present != expected or actual_hash != expected_hash, + } + + +def _transcript_terminal_state(path: Path, agent: str) -> str: + completed = False + failed = False + try: + with path.open(encoding="utf-8") as handle: + for line in handle: + try: + event = json.loads(line) + except json.JSONDecodeError: + failed = True + continue + if not isinstance(event, dict): + continue + if agent == "codex": + if event.get("type") == "turn.completed": + completed = True + if event.get("type") in {"turn.failed", "error"}: + failed = True + else: + if event.get("type") == "result": + terminal_reason = event.get("terminal_reason") + subtype = event.get("subtype") + if terminal_reason == "completed" and subtype == "success": + completed = True + else: + failed = True + except (OSError, UnicodeDecodeError): + return "unreadable" + if completed and not failed: + return "completed" + if completed: + return "completed-with-error-events" + if failed: + return "failed" + return "incomplete" + + +def _claude_catalog_audit( + path: Path, + *, + condition: str, + skill_name: str = SKILL_NAME, +) -> dict[str, Any]: + init: dict[str, Any] | None = None + with path.open(encoding="utf-8") as handle: + for line in handle: + event = json.loads(line) + if ( + isinstance(event, dict) + and event.get("type") == "system" + and event.get("subtype") == "init" + ): + init = event + break + if init is None: + return {"valid": False, "reason": "missing system init event"} + plugins = init.get("plugins") + plugin_names = ( + [item.get("name") for item in plugins if isinstance(item, dict)] + if isinstance(plugins, list) + else [] + ) + raw_skills = init.get("skills") + skills: list[Any] = raw_skills if isinstance(raw_skills, list) else [] + measured_name = f"agent-qualification-treatment:{skill_name}" + measured_present = measured_name in skills + expected_present = condition == "skill" + errors: list[str] = [] + if plugin_names != ["agent-qualification-treatment"]: + errors.append("unexpected plugin catalog") + if init.get("mcp_servers") != []: + errors.append("MCP catalog was not empty") + if measured_present is not expected_present: + errors.append("measured skill catalog presence changed") + required_tools = { + "Bash", + "Edit", + "Glob", + "Grep", + "Read", + "Skill", + "WebFetch", + "WebSearch", + "Write", + } + if set(init.get("tools", [])) != required_tools: + errors.append("tool catalog changed") + return { + "valid": not errors, + "errors": errors, + "plugin_names": plugin_names, + "mcp_servers": init.get("mcp_servers"), + "measured_skill": measured_name, + "measured_skill_expected": expected_present, + "measured_skill_present": measured_present, + "model": init.get("model"), + } + + +def _entry_by_run_id(plan: Mapping[str, Any], run_id: str) -> dict[str, Any]: + entries = plan.get("entries") + if not isinstance(entries, list): + raise EvaluationError("validated plan lost its entries") + matches = [entry for entry in entries if entry.get("run_id") == run_id] + if len(matches) != 1 or not isinstance(matches[0], dict): + raise EvaluationError(f"run {run_id!r} is not one frozen plan entry") + return matches[0] + + +def _run_root(run_id: str) -> Path: + return DEFAULT_RUNS_PATH.resolve() / run_id + + +def _validate_prior_verdicts( + plan: Mapping[str, Any], + freeze: Mapping[str, Any], + ordinal: int, +) -> None: + entries = plan.get("entries") + if not isinstance(entries, list): + raise EvaluationError("validated plan lost its entries") + for entry in entries[: ordinal - 1]: + validate_run(str(entry["run_id"]), plan=plan, freeze=freeze) + + +def _expected_metadata_bindings( + entry: Mapping[str, Any], + *, + freeze: Mapping[str, Any], + plan: Mapping[str, Any], + case: Mapping[str, Any], +) -> dict[str, Any]: + agent = str(entry["agent"]) + binary_binding = freeze["binary_bindings"][agent] + return { + "protocol": PROTOCOL_ID, + "run_id": entry["run_id"], + "evidence_id": entry["evidence_id"], + "ordinal": entry["ordinal"], + "condition": entry["condition"], + "case_id": entry["case_id"], + "agent": agent, + "repetition": entry["repetition"], + "manifest_sha256": baseline._sha256_file(baseline.DEFAULT_MANIFEST_PATH), + "gold_manifest_sha256": str(case["gold_sha256"]), + "baseline_runner_sha256": baseline._sha256_file(baseline.RUNNER_PATH), + "treatment_runner_sha256": baseline._sha256_file(RUNNER_PATH), + "treatment_freeze_sha256": freeze["_sha256"], + "master_plan_sha256": plan["_sha256"], + "protocol_sha256": baseline._sha256_file(baseline.PROTOCOL_PATH), + "prompt_template_sha256": baseline._sha256_file(baseline.PROMPT_PATH), + "docs_treatment_sha256": _sha256_bytes(baseline.DOCS_TREATMENT.encode()), + "rendered_prompt_sha256": entry["rendered_prompt_sha256"], + "snapshot_sha256": str(case["snapshot_sha256"]), + "requested_model": EXPECTED_MODELS[agent], + "binary_sha256": binary_binding["sha256"], + "cli_version": binary_binding["cli_version"], + "timeout_seconds": baseline.DEFAULT_TIMEOUT_SECONDS, + "assigned_payload_is_intervention": False, + "post_launch_intervention_class": "I0", + } + + +def _write_pre_score_verdict( + run_root: Path, + *, + valid: bool, + errors: Sequence[str], + metadata: Mapping[str, Any], +) -> dict[str, Any]: + verdict = { + "kind": VERDICT_KIND, + "protocol": PROTOCOL_ID, + "run_id": metadata.get("run_id"), + "valid": valid, + "errors": list(errors), + "terminal_state": metadata.get("terminal_state"), + "timed_out": metadata.get("timed_out"), + "exit_code": metadata.get("exit_code"), + "metadata_sha256": baseline._sha256_file(run_root / "metadata.json"), + "transcript_sha256": metadata.get("transcript_sha256"), + "created_at": baseline._utc_now(), + } + path = run_root / "pre-score-verdict.json" + if path.exists(): + raise EvaluationError(f"refusing to overwrite run verdict {path}") + path.write_bytes(_json_bytes(verdict)) + return verdict + + +def validate_run( + run_id: str, + *, + plan: Mapping[str, Any] | None = None, + freeze: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + if freeze is None: + freeze = validate_freeze(DEFAULT_FREEZE_PATH) + if plan is None: + raw_plan = freeze.get("_plan") + if not isinstance(raw_plan, dict): + raise EvaluationError("validated freeze lost its master plan") + plan = raw_plan + entry = _entry_by_run_id(plan, run_id) + run_root = _run_root(run_id) + metadata_path = _private_file(run_root / "metadata.json", "run metadata") + metadata, _ = _read_json_once(metadata_path, "run metadata") + manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH) + case = baseline._case_index(manifest)[str(entry["case_id"])] + errors: list[str] = [] + for field, expected in _expected_metadata_bindings( + entry, freeze=freeze, plan=plan, case=case + ).items(): + if metadata.get(field) != expected: + errors.append(f"metadata {field} binding changed") + agent = str(entry["agent"]) + transcript_path = run_root / f"{agent}.jsonl" + stderr_path = run_root / f"{agent}.stderr" + for artifact in (transcript_path, stderr_path, run_root / "prompt.txt"): + if not artifact.is_file() or artifact.is_symlink(): + errors.append(f"missing required run artifact {artifact.name}") + if transcript_path.is_file(): + actual_transcript_hash = baseline._sha256_file(transcript_path) + if metadata.get("transcript_sha256") != actual_transcript_hash: + errors.append("transcript hash changed") + if stderr_path.is_file(): + actual_stderr_hash = baseline._sha256_file(stderr_path) + if metadata.get("stderr_sha256") != actual_stderr_hash: + errors.append("stderr hash changed") + prompt_path = run_root / "prompt.txt" + if prompt_path.is_file() and baseline._sha256_file(prompt_path) != entry.get( + "rendered_prompt_sha256" + ): + errors.append("rendered prompt artifact changed") + if metadata.get("model_identifier") != EXPECTED_MODELS[agent]: + errors.append("reported model identifier changed") + expected_skill = entry["condition"] == "skill" + if metadata.get("skill_injected") is not expected_skill: + errors.append("skill injection flag changed") + if metadata.get("frozen_skill_sha256") != plan["skill"]["sha256"]: + errors.append("frozen skill metadata binding changed") + surface = metadata.get("native_surface") + if not isinstance(surface, dict): + errors.append("native skill surface audit is missing") + else: + expected_kind = ( + "isolated-codex-home-skill-v1" + if agent == "codex" + else "isolated-claude-plugin-skill-v1" + ) + if surface.get("kind") != expected_kind: + errors.append("native skill surface kind changed") + if surface.get("skill_installed") is not expected_skill: + errors.append("native skill surface condition changed") + if surface.get("semantic_file_count") != (1 if expected_skill else 0): + errors.append("native skill semantic file count changed") + expected_skill_hash = plan["skill"]["sha256"] if expected_skill else None + if surface.get("skill_sha256") != expected_skill_hash: + errors.append("native skill surface hash changed") + final_audit = surface.get("final_audit") + if not isinstance(final_audit, dict) or final_audit.get("mutated") is not False: + errors.append("native skill surface final audit failed") + if agent == "claude": + catalog_audit = surface.get("provider_catalog_audit") + if not isinstance(catalog_audit, dict) or catalog_audit.get("valid") is not True: + errors.append("Claude provider catalog audit failed") + terminal_state = metadata.get("terminal_state") + timed_out = metadata.get("timed_out") + if timed_out is not True and terminal_state != "completed": + errors.append("provider subprocess did not produce a completed terminal event") + fixtures = metadata.get("fixtures") + if not isinstance(fixtures, dict): + errors.append("fixture audit is missing") + else: + for name, fixture in fixtures.items(): + if not isinstance(fixture, dict): + errors.append(f"fixture {name} audit is invalid") + continue + if fixture.get("kind") == "none": + continue + if fixture.get("mutated") is not False: + errors.append(f"fixture {name} mutated") + if fixture.get("final_tree_sha256") != fixture.get("initial_tree_sha256"): + errors.append(f"fixture {name} final hash changed") + verdict_path = run_root / "pre-score-verdict.json" + if not verdict_path.is_file(): + raise EvaluationError(f"run {run_id!r} is missing its pre-score verdict") + verdict, _ = _read_json_once(verdict_path, "pre-score verdict") + if verdict.get("kind") != VERDICT_KIND or verdict.get("run_id") != run_id: + errors.append("pre-score verdict identity changed") + if verdict.get("metadata_sha256") != baseline._sha256_file(metadata_path): + errors.append("pre-score verdict metadata hash changed") + if verdict.get("transcript_sha256") != metadata.get("transcript_sha256"): + errors.append("pre-score verdict transcript hash changed") + if verdict.get("valid") is not True or verdict.get("errors") != []: + errors.append("pre-score verdict did not record a valid run") + if errors: + raise EvaluationError("; ".join(errors)) + return verdict + + +def execute_run( + run_id: str, + *, + timeout_seconds: int = baseline.DEFAULT_TIMEOUT_SECONDS, +) -> dict[str, Any]: + if timeout_seconds != baseline.DEFAULT_TIMEOUT_SECONDS: + raise EvaluationError( + "official treatment timeout must remain " f"{baseline.DEFAULT_TIMEOUT_SECONDS} seconds" + ) + freeze = validate_freeze(DEFAULT_FREEZE_PATH) + raw_plan = freeze.get("_plan") + if not isinstance(raw_plan, dict): + raise EvaluationError("validated freeze lost its master plan") + plan = raw_plan + entry = _entry_by_run_id(plan, run_id) + _validate_prior_verdicts(plan, freeze, int(entry["ordinal"])) + + manifest = baseline.validate_manifest(baseline.DEFAULT_MANIFEST_PATH) + case = baseline._case_index(manifest)[str(entry["case_id"])] + if case.get("status") != "ready": + raise EvaluationError(f"case {entry['case_id']!r} is not ready") + run_root = _run_root(run_id) + if run_root.exists() or run_root.is_symlink(): + raise EvaluationError(f"refusing to overwrite existing run {run_root}") + + agent = str(entry["agent"]) + condition = str(entry["condition"]) + source = (REPOSITORY_ROOT / str(case["source"])).resolve(strict=True) + work_root = baseline.CODEX_WORK_ROOT if agent == "codex" else baseline.CLAUDE_WORK_ROOT + external_run_root = work_root / run_id + if external_run_root.exists() or external_run_root.is_symlink(): + raise EvaluationError(f"refusing to reuse external workspace {external_run_root}") + workspace = external_run_root / "workspace" + scratch = external_run_root / "scratch" + transcript_path = run_root / f"{agent}.jsonl" + stderr_path = run_root / f"{agent}.stderr" + + run_root.mkdir(parents=True) + scratch.mkdir(parents=True) + baseline._initialize_snapshot(source, workspace) + prompt = render_prompt(case) + if _sha256_bytes(prompt.encode()) != entry["rendered_prompt_sha256"]: + raise EvaluationError("rendered treatment prompt changed before launch") + (run_root / "prompt.txt").write_text(prompt) + environment = baseline._clean_agent_environment(scratch) + fixtures = baseline._prepare_case_fixtures(case, scratch) + skill_content = plan.get("_skill_content") + if not isinstance(skill_content, bytes): + raise EvaluationError("validated plan lost the frozen skill bytes") + command, native_surface, plugin_root = _prepare_native_surface( + agent=agent, + condition=condition, + workspace=workspace, + external_run_root=external_run_root, + scratch=scratch, + environment=environment, + skill_content=skill_content, + ) + settings_path: Path | None = None + if agent == "claude": + if plugin_root is None: + raise EvaluationError("Claude native plugin root was not prepared") + settings_path = run_root / "claude-settings.json" + baseline._write_json( + settings_path, + _claude_treatment_settings(workspace, scratch, plugin_root), + ) + command = _claude_treatment_command(settings_path, plugin_root) + + binary = baseline.CODEX_BINARY if agent == "codex" else baseline.CLAUDE_BINARY + cli_version = baseline._binary_version(binary, environment) + started_at = baseline._utc_now() + bindings = _expected_metadata_bindings(entry, freeze=freeze, plan=plan, case=case) + metadata: dict[str, Any] = { + **bindings, + "started_at": started_at, + "timeout_seconds": timeout_seconds, + "command": command, + "prompt_transport": "stdin", + "cli_version": cli_version, + "binary_sha256": baseline._sha256_file(binary), + "removed_environment_names": sorted(baseline.SECRET_ENVIRONMENT_NAMES), + "fixtures": fixtures, + "native_surface": native_surface, + "skill_injected": condition == "skill", + "frozen_skill_sha256": plan["skill"]["sha256"], + "treatment_payload_transport": "native local skill mechanism", + } + baseline._write_json(run_root / "metadata.json", metadata) + + exit_code, timed_out, elapsed_seconds = baseline._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript_path, + stderr_path=stderr_path, + timeout_seconds=timeout_seconds, + ) + native_surface["final_audit"] = _audit_native_surface( + agent=agent, + condition=condition, + external_run_root=external_run_root, + scratch=scratch, + skill_content=skill_content, + ) + baseline._write_workspace_evidence(workspace, run_root) + baseline._record_final_fixture_state(fixtures, scratch) + shutil.copytree( + workspace, + run_root / "workspace-final", + symlinks=True, + ignore=shutil.ignore_patterns( + ".git", "bin", "obj", "target", "node_modules", ".eval-cache" + ), + ) + terminal_state = _transcript_terminal_state(transcript_path, agent) + if agent == "claude": + native_surface["provider_catalog_audit"] = _claude_catalog_audit( + transcript_path, + condition=condition, + ) + metadata.update( + { + "completed_at": baseline._utc_now(), + "exit_code": exit_code, + "timed_out": timed_out, + "elapsed_seconds": elapsed_seconds, + "terminal_state": terminal_state, + "transcript_sha256": baseline._sha256_file(transcript_path), + "stderr_sha256": baseline._sha256_file(stderr_path), + "fixtures": fixtures, + } + ) + model_identifier, model_identifier_source = baseline._reported_model(transcript_path, agent) + metadata["model_identifier"] = model_identifier + metadata["model_identifier_source"] = model_identifier_source + baseline._write_json(run_root / "metadata.json", metadata) + + pre_errors: list[str] = [] + if model_identifier != EXPECTED_MODELS[agent]: + pre_errors.append(f"reported model {model_identifier!r} != {EXPECTED_MODELS[agent]!r}") + if timed_out is not True and terminal_state != "completed": + pre_errors.append(f"provider terminal state was {terminal_state!r}") + if native_surface["final_audit"]["mutated"] is not False: + pre_errors.append("native skill surface mutated") + if agent == "claude" and native_surface["provider_catalog_audit"]["valid"] is not True: + pre_errors.append("Claude provider catalog isolation failed") + for name, fixture in fixtures.items(): + if fixture.get("kind") != "none" and fixture.get("mutated") is not False: + pre_errors.append(f"fixture {name} mutated") + verdict = _write_pre_score_verdict( + run_root, + valid=not pre_errors, + errors=pre_errors, + metadata=metadata, + ) + if pre_errors: + raise EvaluationError(f"run {run_id!r} is technically invalid: " + "; ".join(pre_errors)) + validate_run(run_id, plan=plan, freeze=freeze) + shutil.rmtree(external_run_root) + return {"metadata": metadata, "verdict": verdict} + + +def _transcript_result_text(path: Path, agent: str) -> str: + result = "" + with path.open(encoding="utf-8") as handle: + for line in handle: + event = json.loads(line) + if not isinstance(event, dict): + continue + if agent == "codex" and event.get("type") == "item.completed": + item = event.get("item") + if isinstance(item, dict) and item.get("type") == "agent_message": + text = item.get("text") + if isinstance(text, str): + result = text + if agent == "claude" and event.get("type") == "result": + text = event.get("result") + if isinstance(text, str): + result = text + return result.strip() + + +def run_native_delivery_shakedown( + *, + output_path: Path = DEFAULT_SHAKEDOWN_VERDICT_PATH, +) -> dict[str, Any]: + prior_root = PRIVATE_ROOT / "shakedowns" / "native-skill-delivery-20260729" + skill_source = _private_file( + prior_root / "skill-source" / "SKILL.md", "synthetic shakedown skill" + ) + skill_content = skill_source.read_bytes() + skill_hash = _sha256_bytes(skill_content) + marker = "NATIVE_SKILL_PROBE_20260729_8C41D6" + if marker.encode() not in skill_content: + raise EvaluationError("synthetic shakedown skill lost its marker") + verified_root = PRIVATE_ROOT / "shakedowns" / "native-skill-delivery-verified-20260729" + if verified_root.exists() or verified_root.is_symlink(): + raise EvaluationError(f"refusing to reuse shakedown root {verified_root}") + verified_root.mkdir(parents=True) + prompt = ( + "Run the native-delivery-probe shakedown now. If its native skill is " + "available, use it. If unavailable, reply exactly " + "NO_NATIVE_DELIVERY. Do not search for skill files or inspect any path " + "not supplied by the native skill catalog." + ) + providers: dict[str, Any] = {} + for agent in ("codex", "claude"): + agent_result: dict[str, Any] = { + "synthetic_skill_sha256": skill_hash, + } + for condition, label in (("docs", "control"), ("skill", "treatment")): + root = verified_root / f"{agent}-{label}" + workspace = root / "workspace" + scratch = root / "scratch" + workspace.mkdir(parents=True) + scratch.mkdir(parents=True) + (workspace / "README.md").write_text("Synthetic native skill delivery probe.\n") + environment = baseline._clean_agent_environment(scratch) + command, surface, plugin_root = _prepare_native_surface( + agent=agent, + condition=condition, + workspace=workspace, + external_run_root=root, + scratch=scratch, + environment=environment, + skill_content=skill_content, + skill_name="native-delivery-probe", + ) + if agent == "claude": + if plugin_root is None: + raise EvaluationError("Claude shakedown plugin was not prepared") + settings_path = root / "claude-settings.json" + baseline._write_json( + settings_path, + _claude_treatment_settings(workspace, scratch, plugin_root), + ) + command = _claude_treatment_command(settings_path, plugin_root) + transcript = root / f"{agent}.jsonl" + stderr = root / f"{agent}.stderr" + exit_code, timed_out, elapsed = baseline._run_subprocess( + command, + cwd=workspace, + environment=environment, + prompt_stdin=prompt, + stdout_path=transcript, + stderr_path=stderr, + timeout_seconds=600, + ) + surface["final_audit"] = _audit_native_surface( + agent=agent, + condition=condition, + external_run_root=root, + scratch=scratch, + skill_content=skill_content, + skill_name="native-delivery-probe", + ) + if agent == "claude": + surface["provider_catalog_audit"] = _claude_catalog_audit( + transcript, + condition=condition, + skill_name="native-delivery-probe", + ) + terminal_state = _transcript_terminal_state(transcript, agent) + result = _transcript_result_text(transcript, agent) + model, model_source = baseline._reported_model(transcript, agent) + expected_result = marker if condition == "skill" else "NO_NATIVE_DELIVERY" + if exit_code != 0 or timed_out or terminal_state != "completed": + raise EvaluationError(f"{agent} {label} shakedown did not complete cleanly") + if surface["final_audit"]["mutated"] is not False: + raise EvaluationError(f"{agent} {label} native skill surface mutated") + if agent == "claude" and surface["provider_catalog_audit"]["valid"] is not True: + raise EvaluationError(f"{agent} {label} provider catalog was not isolated") + if result != expected_result: + raise EvaluationError( + f"{agent} {label} shakedown returned {result!r}, " + f"expected {expected_result!r}" + ) + if model != EXPECTED_MODELS[agent]: + raise EvaluationError(f"{agent} shakedown model changed: {model!r}") + agent_result[f"{label}_result"] = result + agent_result[f"{label}_transcript"] = _file_record(transcript) + agent_result[f"{label}_stderr"] = _file_record(stderr) + agent_result[f"{label}_surface"] = surface + agent_result[f"{label}_elapsed_seconds"] = elapsed + agent_result[f"{label}_model"] = model + agent_result[f"{label}_model_source"] = model_source + agent_result["passed"] = True + providers[agent] = agent_result + verdict = { + "kind": SHAKEDOWN_KIND, + "protocol": PROTOCOL_ID, + "created_at": baseline._utc_now(), + "passed": True, + "runner": _file_record(RUNNER_PATH), + "marker": marker, + "synthetic_skill": _file_record(skill_source), + "synthetic_skill_sha256": skill_hash, + "prompt_sha256": _sha256_bytes(prompt.encode()), + "providers": providers, + } + _write_new_json(output_path, verdict, "native-delivery shakedown verdict") + return verdict + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("shakedown", help="verify native skill delivery on a synthetic case") + + plan = subparsers.add_parser("plan", help="create the immutable 24-run plan") + plan.add_argument("--output", type=Path, default=DEFAULT_PLAN_PATH) + + freeze = subparsers.add_parser("freeze", help="freeze all treatment inputs") + freeze.add_argument("--output", type=Path, default=DEFAULT_FREEZE_PATH) + + subparsers.add_parser("validate", help="validate the complete treatment freeze") + + run = subparsers.add_parser("run", help="execute the next frozen plan entry") + run.add_argument("--run-id", required=True) + run.add_argument("--timeout-seconds", type=int, default=baseline.DEFAULT_TIMEOUT_SECONDS) + + validate_run_parser = subparsers.add_parser( + "validate-run", help="validate one completed pre-score run" + ) + validate_run_parser.add_argument("--run-id", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + try: + if args.command == "shakedown": + payload = run_native_delivery_shakedown() + elif args.command == "plan": + payload = create_plan( + baseline.DEFAULT_MANIFEST_PATH, + skill_path=DEFAULT_SKILL_PATH, + external_pins_path=DEFAULT_EXTERNAL_PINS_PATH, + protocol_amendment_path=DEFAULT_PROTOCOL_AMENDMENT_PATH, + authorization_path=DEFAULT_AUTHORIZATION_PATH, + claude_wrapper_path=DEFAULT_CLAUDE_WRAPPER_PATH, + output_path=args.output, + ) + elif args.command == "freeze": + payload = create_freeze( + args.output, + plan_path=DEFAULT_PLAN_PATH, + shakedown_verdict_path=DEFAULT_SHAKEDOWN_VERDICT_PATH, + ) + elif args.command == "validate": + payload = validate_freeze(DEFAULT_FREEZE_PATH) + elif args.command == "run": + payload = execute_run(args.run_id, timeout_seconds=args.timeout_seconds) + elif args.command == "validate-run": + payload = validate_run(args.run_id) + else: # pragma: no cover - argparse enforces the command. + raise EvaluationError(f"unknown command {args.command!r}") + except (EvaluationError, OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + printable = { + key: value + for key, value in payload.items() + if not key.startswith("_") and not isinstance(value, bytes) + } + print(json.dumps(printable, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 966bc6f..6d6eae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,9 @@ addopts = "-q" [tool.black] line-length = 100 target-version = ["py310"] +# This runner is hash-bound by the frozen v2 cohort and cannot be reformatted +# without invalidating the preregistration. +extend-exclude = "^/experiments/agent_qualification_generalization.py$" [tool.mypy] python_version = "3.10" @@ -103,6 +106,7 @@ check_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true warn_unused_configs = true +exclude = "^experiments/private/" [tool.coverage.run] source = ["tracebook"] diff --git a/tests/test_agent_qualification_experiment.py b/tests/test_agent_qualification_experiment.py new file mode 100644 index 0000000..bf3564c --- /dev/null +++ b/tests/test_agent_qualification_experiment.py @@ -0,0 +1,344 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from experiments import agent_qualification + + +def _file_hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _manifest(tmp_path: Path, *, status: str = "ready") -> Path: + source = tmp_path / "source" + source.mkdir(parents=True) + (source / "engine.txt").write_text("price-time\n") + gold = tmp_path / "gold.json" + gold.write_text('{"terminal_class":"conformant"}\n') + manifest = { + "protocol": agent_qualification.PROTOCOL_ID, + "protocol_sha256": _file_hash(agent_qualification.PROTOCOL_PATH), + "prompt_sha256": _file_hash(agent_qualification.PROMPT_PATH), + "cases": [ + { + "id": "c1", + "source": str(source), + "snapshot_id": "snapshot-c1", + "revision": "revision-c1", + "commands": "test-engine", + "declared_claim": "FIFO matching", + "excluded_scope": "latency", + "snapshot_sha256": agent_qualification._snapshot_digest(source), + "gold_manifest": str(gold), + "gold_sha256": _file_hash(gold), + "dependency_cache": {"kind": "none"}, + "native_test_tool": {"kind": "none"}, + "status": status, + } + ], + } + path = tmp_path / "cases.json" + path.write_text(json.dumps(manifest)) + return path + + +def _add_frozen_fixture( + manifest_path: Path, + fixture_root: Path, + *, + field_name: str, + kind: str, +) -> None: + source = fixture_root / "source" + source.mkdir(parents=True) + (source / "package.bin").write_bytes(b"frozen dependency") + case = json.loads(manifest_path.read_text())["cases"][0] + inventory = agent_qualification._tree_inventory(source) + tree_hash = agent_qualification._snapshot_digest(source) + fixture_manifest = { + "kind": kind, + "case_id": case["id"], + "snapshot_sha256": case["snapshot_sha256"], + "tree_sha256": tree_hash, + "file_count": len(inventory), + "total_bytes": sum(item["bytes"] for item in inventory), + "files": inventory, + } + fixture_manifest_path = fixture_root / "manifest.json" + fixture_manifest_path.write_text(json.dumps(fixture_manifest)) + payload = json.loads(manifest_path.read_text()) + payload["cases"][0][field_name] = { + "kind": kind, + "source": str(source), + "manifest": str(fixture_manifest_path), + "manifest_sha256": _file_hash(fixture_manifest_path), + "tree_sha256": tree_hash, + } + manifest_path.write_text(json.dumps(payload)) + + +def test_validate_manifest_rejects_snapshot_drift(tmp_path): + manifest_path = _manifest(tmp_path) + manifest = agent_qualification.validate_manifest(manifest_path) + assert manifest["protocol"] == agent_qualification.PROTOCOL_ID + + source = Path(manifest["cases"][0]["source"]) + (source / "engine.txt").write_text("changed\n") + + with pytest.raises(agent_qualification.EvaluationError, match="snapshot changed"): + agent_qualification.validate_manifest(manifest_path) + + +def test_validate_manifest_rejects_gold_drift(tmp_path): + manifest_path = _manifest(tmp_path) + manifest = agent_qualification.validate_manifest(manifest_path) + + Path(manifest["cases"][0]["gold_manifest"]).write_text( + '{"terminal_class":"candidate-defect"}\n' + ) + + with pytest.raises(agent_qualification.EvaluationError, match="gold manifest changed"): + agent_qualification.validate_manifest(manifest_path) + + +def test_frozen_cache_copy_is_verified_and_isolated(tmp_path, monkeypatch): + manifest_path = _manifest(tmp_path) + _add_frozen_fixture( + manifest_path, + tmp_path / "fixtures" / "nuget", + field_name="dependency_cache", + kind="nuget-global-packages-v1", + ) + monkeypatch.setattr(agent_qualification, "PRIVATE_ROOT", tmp_path) + + manifest = agent_qualification.validate_manifest(manifest_path) + case = manifest["cases"][0] + scratch = tmp_path / "scratch" + agent_qualification._clean_agent_environment(scratch) + prepared = agent_qualification._prepare_case_fixtures(case, scratch) + + assert ( + prepared["dependency_cache"]["initial_tree_sha256"] + == case["dependency_cache"]["tree_sha256"] + ) + copied = scratch / "nuget" / "package.bin" + copied.write_bytes(b"mutated run copy") + source = Path(case["dependency_cache"]["source"]) / "package.bin" + assert source.read_bytes() == b"frozen dependency" + agent_qualification._record_final_fixture_state(prepared, scratch) + assert prepared["dependency_cache"]["mutated"] is True + + +def test_frozen_cache_rejects_tree_drift(tmp_path, monkeypatch): + manifest_path = _manifest(tmp_path) + fixture_root = tmp_path / "fixtures" / "nuget" + _add_frozen_fixture( + manifest_path, + fixture_root, + field_name="dependency_cache", + kind="nuget-global-packages-v1", + ) + monkeypatch.setattr(agent_qualification, "PRIVATE_ROOT", tmp_path) + (fixture_root / "source" / "package.bin").write_bytes(b"drift") + + with pytest.raises(agent_qualification.EvaluationError, match="manifest binding"): + agent_qualification.validate_manifest(manifest_path) + + +def test_render_prompt_keeps_baseline_blind(): + case = { + "id": "c1", + "snapshot_id": "snapshot-c1", + "revision": "revision-c1", + "commands": "test-engine", + "declared_claim": "FIFO matching", + "excluded_scope": "latency", + } + + baseline = agent_qualification.render_prompt(case, "baseline") + docs = agent_qualification.render_prompt(case, "docs") + + assert "tracebook" not in baseline.lower() + assert "tracebook-conformance==0.6.0" in docs + assert "{{" not in baseline + assert "snapshot-c1" in baseline + + +def test_plan_is_seeded_and_excludes_pending_cases(tmp_path): + manifest_path = _manifest(tmp_path) + pending_path = _manifest(tmp_path / "pending", status="pending-gold") + first_path = tmp_path / "first.json" + second_path = tmp_path / "second.json" + pending_plan_path = tmp_path / "pending-plan.json" + + first = agent_qualification.create_plan( + manifest_path, first_path, repetitions=3, conditions=("baseline",) + ) + second = agent_qualification.create_plan( + manifest_path, second_path, repetitions=3, conditions=("baseline",) + ) + pending = agent_qualification.create_plan( + pending_path, + pending_plan_path, + repetitions=3, + conditions=("baseline",), + ) + + assert first["entries"] == second["entries"] + assert len(first["entries"]) == 6 + assert {entry["agent"] for entry in first["entries"]} == {"codex", "claude"} + assert pending["entries"] == [] + + +def test_skill_condition_is_locked_before_baseline(tmp_path): + manifest_path = _manifest(tmp_path) + + with pytest.raises(agent_qualification.EvaluationError, match="remain locked"): + agent_qualification.execute_run( + manifest_path, + case_id="c1", + agent="codex", + repetition=1, + condition="skill", + runs_path=tmp_path / "runs", + timeout_seconds=1, + ) + + +def test_claude_permissions_delegate_paths_to_fail_closed_sandbox(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + workspace.mkdir() + scratch.mkdir() + + settings = agent_qualification._claude_settings(workspace, scratch) + + assert settings["permissions"]["allow"] == [ + f"Read(/{workspace}/**)", + f"Edit(/{workspace}/**)", + "Bash", + "WebSearch", + "WebFetch", + ] + assert f"Read(/{Path.home()}/**)" in settings["permissions"]["deny"] + assert f"Edit(/{workspace}/.git/**)" in settings["permissions"]["deny"] + assert str(workspace) in settings["sandbox"]["filesystem"]["allowWrite"] + assert str(workspace / ".git") in settings["sandbox"]["filesystem"]["denyWrite"] + assert str(Path.home()) in settings["sandbox"]["filesystem"]["denyRead"] + for candidate in ( + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + ): + assert (str(candidate) in settings["sandbox"]["filesystem"]["allowRead"]) is ( + candidate.exists() + ) + _, dotnet_launcher_dir, dotnet_root = agent_qualification._dotnet_layout() + if dotnet_launcher_dir is None: + assert "DOTNET_ROOT" not in settings["env"] + else: + assert settings["env"]["PATH"].startswith(str(dotnet_launcher_dir)) + assert settings["env"]["DOTNET_ROOT"] == str(dotnet_root) + assert settings["env"]["NuGetAudit"] == "false" + + +def test_dotnet_environment_preseeds_nuget_migration_state(tmp_path): + environment = agent_qualification._dotnet_environment(tmp_path) + + xdg_data_home = tmp_path / "xdg-data" + assert environment["XDG_DATA_HOME"] == str(xdg_data_home) + assert (xdg_data_home / "NuGet" / "Migrations" / "1").is_file() + + +def test_codex_command_uses_pure_permission_profile(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + workspace.mkdir() + scratch.mkdir() + + command = agent_qualification._codex_command(workspace, scratch) + permission_override = next( + item for item in command if item.startswith("permissions.agent-eval=") + ) + + assert command[-1] == "-" + assert "--sandbox" not in command + assert "workspace-write" not in command + assert command.index("exec") > command.index("--disable") + assert 'default_permissions="agent-eval"' in command + assert '":root"="deny"' in permission_override + assert '":workspace_roots"={"."="write",".git"="read"}' in permission_override + assert f'{scratch}"="write"' in permission_override + for candidate in ( + Path("/usr/local/bin/dotnet"), + Path("/usr/local/share/dotnet"), + Path("/usr/local/opt"), + Path("/usr/local/Cellar"), + ): + assert (f'"{candidate}"="read"' in permission_override) is candidate.exists() + assert 'mode="limited"' in permission_override + assert "allow_upstream_proxy=false" in permission_override + assert "allow_local_binding=true" not in permission_override + assert '"pypi.org"="allow"' in permission_override + assert "nuget.org" not in permission_override + assert 'shell_environment_policy.inherit="core"' in command + assert any(item.startswith("shell_environment_policy.set=") for item in command) + assert "--search" in command + shell_environment = agent_qualification._codex_shell_environment(scratch) + _, dotnet_launcher_dir, dotnet_root = agent_qualification._dotnet_layout() + if dotnet_launcher_dir is None: + assert "DOTNET_ROOT" not in shell_environment + else: + assert shell_environment["PATH"].startswith(str(dotnet_launcher_dir)) + assert shell_environment["DOTNET_ROOT"] == str(dotnet_root) + + +def test_clean_environment_removes_secret_shaped_names(tmp_path, monkeypatch): + monkeypatch.setenv("AGENT_EVAL_SAFE_VALUE", "safe") + monkeypatch.setenv("UNLISTED_PRIVATE_TOKEN", "secret") + monkeypatch.setenv("SSH_AUTH_SOCK", "/private/tmp/agent.sock") + + environment = agent_qualification._clean_agent_environment(tmp_path) + + assert environment["AGENT_EVAL_SAFE_VALUE"] == "safe" + assert "UNLISTED_PRIVATE_TOKEN" not in environment + assert "SSH_AUTH_SOCK" not in environment + + +def test_codex_home_is_outside_agent_writable_scratch(tmp_path, monkeypatch): + external_root = tmp_path / "run" + scratch = external_root / "scratch" + external_root.mkdir() + scratch.mkdir() + auth = tmp_path / "auth.json" + auth.write_text("{}\n") + monkeypatch.setattr(agent_qualification, "CODEX_AUTH_PATH", auth) + environment: dict[str, str] = {} + + codex_home = agent_qualification._prepare_codex_environment(external_root, scratch, environment) + + assert codex_home == external_root / "codex-home" + assert scratch not in codex_home.parents + assert (codex_home / "auth.json").is_symlink() + assert Path(environment["HOME"]) == scratch / "shell-home" + assert Path(environment["CODEX_HOME"]) == codex_home + assert Path(environment["CODEX_SQLITE_HOME"]) == codex_home / "state" + + +def test_reported_model_uses_claude_transcript_and_codex_command(tmp_path): + transcript = tmp_path / "events.jsonl" + transcript.write_text( + '{"type":"system","model":"claude-opus-exact"}\n' '{"type":"result","result":"done"}\n' + ) + + assert agent_qualification._reported_model(transcript, "claude") == ( + "claude-opus-exact", + "transcript", + ) + assert agent_qualification._reported_model(transcript, "codex") == ( + "gpt-5.6-sol", + "command", + ) diff --git a/tests/test_agent_qualification_generalization.py b/tests/test_agent_qualification_generalization.py new file mode 100644 index 0000000..8aed573 --- /dev/null +++ b/tests/test_agent_qualification_generalization.py @@ -0,0 +1,143 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from experiments import agent_qualification_generalization as generalization + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_plan_is_seeded_complete_and_excludes_v1_cases(): + manifest = { + "cases": [ + {"id": "c4-compatible", "status": "ready"}, + {"id": "c5-boundary", "status": "ready"}, + ] + } + + first = generalization.create_plan(manifest) + second = generalization.create_plan(manifest) + + assert first == second + assert first["entry_count"] == 24 + assert [entry["order"] for entry in first["entries"]] == list(range(1, 25)) + assert {entry["agent"] for entry in first["entries"]} == {"codex", "claude"} + assert {entry["condition"] for entry in first["entries"]} == {"docs", "skill"} + assert {entry["case_id"] for entry in first["entries"]} == { + "c4-compatible", + "c5-boundary", + } + assert all("c1" not in entry["id"] and "c3" not in entry["id"] for entry in first["entries"]) + + +def test_pending_cases_are_not_scheduled(): + manifest = { + "cases": [ + {"id": "ready", "status": "ready"}, + {"id": "pending", "status": "pending-gold"}, + ] + } + + plan = generalization.create_plan(manifest) + + assert plan["entry_count"] == 12 + assert {entry["case_id"] for entry in plan["entries"]} == {"ready"} + + +def test_docs_and_skill_prompts_are_text_identical_before_skill_injection(tmp_path, monkeypatch): + prompt = tmp_path / "prompt.txt" + prompt.write_text( + "case={{case_id}}\n" + "snapshot={{snapshot_id}}\n" + "revision={{revision}}\n" + "commands={{commands}}\n" + "claim={{declared_claim}}\n" + "excluded={{excluded_scope}}\n" + ) + monkeypatch.setattr(generalization, "PROMPT_PATH", prompt) + case = { + "id": "heldout", + "snapshot_id": "sha256:abc", + "revision": "pinned", + "commands": "native-test", + "declared_claim": "price-time", + "excluded_scope": "latency", + } + + docs = generalization.render_prompt(case, "docs") + skill = generalization.render_prompt(case, "skill") + + assert docs == skill + assert "tracebook-conformance==0.6.0" in docs + assert "{{" not in docs + + +def test_snapshot_digest_binds_file_mode_and_content(tmp_path): + source = tmp_path / "source" + source.mkdir() + file_path = source / "engine.txt" + file_path.write_text("price-time\n") + first = generalization._snapshot_digest(source) + + file_path.write_text("changed\n") + second = generalization._snapshot_digest(source) + file_path.write_text("price-time\n") + file_path.chmod(0o755) + third = generalization._snapshot_digest(source) + + assert len({first, second, third}) == 3 + + +def test_validate_tree_rejects_inventory_drift(tmp_path, monkeypatch): + private = tmp_path / "private" + source = private / "cache" + source.mkdir(parents=True) + (source / "dependency.bin").write_bytes(b"frozen") + manifests = private / "manifests" + manifests.mkdir() + snapshot_sha256 = "candidate-tree" + inventory = generalization._tree_inventory(source) + tree_sha256 = generalization._snapshot_digest(source) + tree_manifest = { + "kind": "frozen-tree-v1", + "case_id": "case", + "name": "dependency-cache", + "snapshot_sha256": snapshot_sha256, + "tree_sha256": tree_sha256, + "file_count": len(inventory), + "total_bytes": sum(item["bytes"] for item in inventory), + "files": inventory, + } + manifest_path = manifests / "tree.json" + manifest_path.write_text(json.dumps(tree_manifest)) + monkeypatch.setattr(generalization, "REPOSITORY_ROOT", tmp_path) + monkeypatch.setattr(generalization, "PRIVATE_ROOT", private) + case = {"id": "case", "snapshot_sha256": snapshot_sha256} + declaration = { + "kind": "frozen-tree-v1", + "source": "private/cache", + "manifest": "private/manifests/tree.json", + "manifest_sha256": _sha256(manifest_path), + "tree_sha256": tree_sha256, + } + + generalization._validate_tree(case, declaration, name="dependency-cache") + (source / "dependency.bin").write_bytes(b"drift") + + with pytest.raises(generalization.GeneralizationError, match="inventory drifted"): + generalization._validate_tree(case, declaration, name="dependency-cache") + + +def test_freeze_refuses_to_overwrite_existing_bindings(tmp_path, monkeypatch): + manifest = tmp_path / "cases.json" + manifest.write_text("{}\n") + monkeypatch.setattr(generalization, "MANIFEST_PATH", manifest) + monkeypatch.setattr(generalization, "PLAN_PATH", tmp_path / "plan.json") + monkeypatch.setattr(generalization, "FREEZE_PATH", tmp_path / "freeze.json") + + with pytest.raises(generalization.GeneralizationError, match="refusing to overwrite"): + generalization.freeze() diff --git a/tests/test_agent_qualification_generalization_runs.py b/tests/test_agent_qualification_generalization_runs.py new file mode 100644 index 0000000..fc9883a --- /dev/null +++ b/tests/test_agent_qualification_generalization_runs.py @@ -0,0 +1,298 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from experiments import agent_qualification_generalization_runs as execution + + +def test_plan_lookup_uses_exact_frozen_id(): + plan = {"entries": [{"id": "docs__case__codex__r1", "order": 1}]} + + assert execution._plan_entry(plan, "docs__case__codex__r1")["order"] == 1 + with pytest.raises(execution.ExecutionError, match="not one frozen plan entry"): + execution._plan_entry(plan, "missing") + + +def test_authorization_is_provider_and_case_specific(tmp_path, monkeypatch): + auth_root = tmp_path / "authorizations" + statement = "I approve C4 and C5 for Codex." + payload = { + "kind": execution.AUTHORIZATION_KIND, + "protocol": execution.PROTOCOL_ID, + "provider": "codex", + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + "user_statement": statement, + "statement_sha256": hashlib.sha256(statement.encode()).hexdigest(), + "recorded_at": "2026-08-01T00:00:00Z", + } + auth_root.mkdir() + (auth_root / "codex.json").write_text(json.dumps(payload)) + monkeypatch.setattr(execution, "AUTHORIZATION_ROOT", auth_root) + + assert execution.validate_authorization("codex")["provider"] == "codex" + with pytest.raises(execution.ExecutionError, match="cannot read claude authorization"): + execution.validate_authorization("claude") + + +def test_authorization_rejects_statement_drift(tmp_path, monkeypatch): + auth_root = tmp_path / "authorizations" + auth_root.mkdir() + payload = { + "kind": execution.AUTHORIZATION_KIND, + "protocol": execution.PROTOCOL_ID, + "provider": "codex", + "case_ids": ["c4-compatible", "c5-boundary"], + "materials": [ + "public repository snapshots", + "frozen prompt", + "docs-assisted treatment", + "skill-assisted treatment", + ], + "scope": "all-preregistered-runs-and-genuine-technical-restarts", + "user_statement": "changed", + "statement_sha256": "not-the-hash", + "recorded_at": "2026-08-01T00:00:00Z", + } + (auth_root / "codex.json").write_text(json.dumps(payload)) + monkeypatch.setattr(execution, "AUTHORIZATION_ROOT", auth_root) + + with pytest.raises(execution.ExecutionError, match="statement binding drifted"): + execution.validate_authorization("codex") + + +def test_copy_fixture_starts_from_frozen_hash(tmp_path, monkeypatch): + repository = tmp_path / "repo" + source = repository / "private" / "cache" + source.mkdir(parents=True) + (source / "artifact").write_text("frozen") + tree_hash = execution.helpers._snapshot_digest(source) + scratch = tmp_path / "scratch" + scratch.mkdir() + monkeypatch.setattr(execution, "REPOSITORY_ROOT", repository) + case = { + "id": "c5-boundary", + "dependency_cache": { + "source": "private/cache", + "tree_sha256": tree_hash, + "manifest_sha256": "manifest", + }, + } + + fixture = execution._copy_fixture(case, scratch) + + assert fixture["initial_tree_sha256"] == tree_hash + assert Path(fixture["target"]).name == "cargo-home" + assert execution.helpers._snapshot_digest(Path(fixture["target"])) == tree_hash + + +def test_native_surface_has_no_skill_in_docs_condition(tmp_path, monkeypatch): + root = tmp_path / "run" + workspace = root / "workspace" + scratch = root / "scratch" + workspace.mkdir(parents=True) + scratch.mkdir() + monkeypatch.setattr( + execution, + "_configure_delivery", + lambda: None, + ) + monkeypatch.setattr( + execution.delivery, + "_prepare_native_surface", + lambda **kwargs: ( + ["provider"], + { + "skill_installed": kwargs["condition"] == "skill", + "semantic_file_count": 1 if kwargs["condition"] == "skill" else 0, + }, + None, + ), + ) + + _, surface, _ = execution._native_surface( + provider="codex", + condition="docs", + workspace=workspace, + run_root=root, + scratch=scratch, + environment={}, + skill=b"skill", + skill_name="probe", + ) + + assert surface == {"skill_installed": False, "semantic_file_count": 0} + + +def test_validate_prior_runs_enforces_seeded_order(monkeypatch): + seen = [] + monkeypatch.setattr(execution, "validate_run", seen.append) + plan = { + "entries": [ + {"id": "first"}, + {"id": "second"}, + {"id": "third"}, + ] + } + + execution._validate_prior_runs(plan, 3) + + assert seen == ["first", "second"] + + +def test_execution_refuses_without_provider_authorization(monkeypatch): + monkeypatch.setattr(execution, "validate_freeze", lambda: {}) + monkeypatch.setattr( + execution, + "_load_primary", + lambda: ( + {"cases": []}, + { + "entries": [ + { + "id": "skill__c4-compatible__codex__r1", + "agent": "codex", + "case_id": "c4-compatible", + "condition": "skill", + "repetition": 1, + "order": 1, + } + ] + }, + {}, + ), + ) + monkeypatch.setattr( + execution, + "validate_authorization", + lambda provider: (_ for _ in ()).throw(execution.ExecutionError("authorization missing")), + ) + + with pytest.raises(execution.ExecutionError, match="authorization missing"): + execution.execute_run("skill__c4-compatible__codex__r1") + + +def test_wrapper_is_deterministic_and_refuses_drift(tmp_path, monkeypatch): + wrapper = tmp_path / ".claude-plugin" / "plugin.json" + monkeypatch.setattr(execution, "CLAUDE_WRAPPER_PATH", wrapper) + + execution._write_wrapper() + first = wrapper.read_bytes() + execution._write_wrapper() + wrapper.write_text("{}") + + assert json.loads(first) == execution._PLUGIN_MANIFEST + with pytest.raises(execution.ExecutionError, match="wrapper drifted"): + execution._write_wrapper() + + +def test_toolchain_environment_prepends_frozen_tools(tmp_path, monkeypatch): + maven = tmp_path / "maven" + java = tmp_path / "java" + python = tmp_path / "python" + uv = tmp_path / "uv" + for root in (maven, java, python): + (root / "bin").mkdir(parents=True) + uv.write_text("binary") + monkeypatch.setenv("TRACEBOOK_V2_MAVEN_HOME", str(maven)) + monkeypatch.setenv("TRACEBOOK_V2_JAVA_HOME", str(java)) + monkeypatch.setenv("TRACEBOOK_V2_PYTHON_HOME", str(python)) + monkeypatch.setenv("TRACEBOOK_V2_UV_BINARY", str(uv)) + monkeypatch.setenv("PATH", "/usr/bin:/bin") + + environment = execution._toolchain_environment() + + assert environment["JAVA_HOME"] == str(java) + assert environment["PATH"].split(":")[:4] == [ + str(maven / "bin"), + str(java / "bin"), + str(python / "bin"), + str(tmp_path), + ] + + +def test_provider_shells_receive_fresh_paths(tmp_path, monkeypatch): + monkeypatch.delenv("TRACEBOOK_V2_MAVEN_HOME", raising=False) + monkeypatch.delenv("TRACEBOOK_V2_JAVA_HOME", raising=False) + monkeypatch.delenv("TRACEBOOK_V2_PYTHON_HOME", raising=False) + monkeypatch.delenv("TRACEBOOK_V2_UV_BINARY", raising=False) + + values = execution._codex_shell_environment(tmp_path) + + assert values["FRESH_M2"] == str(tmp_path / "m2-repository") + assert values["FRESH_CARGO_HOME"] == str(tmp_path / "cargo-home") + assert values["FRESH_TARGET"] == str(tmp_path / "cargo-target") + + +def test_claude_binding_probes_an_explicit_first_party_route(tmp_path, monkeypatch): + binary = tmp_path / "claude" + binary.write_text("claude") + observed = {} + + monkeypatch.setattr(execution, "_provider_binary", lambda provider: binary) + monkeypatch.setattr(execution.helpers, "_binary_version", lambda *args: "2.1.220") + + def run(command, **kwargs): + observed["command"] = command + return execution.subprocess.CompletedProcess( + command, + 0, + stdout=json.dumps( + { + "loggedIn": True, + "authMethod": "claude.ai", + "apiProvider": "firstParty", + "subscriptionType": "max", + } + ), + ) + + monkeypatch.setattr(execution.subprocess, "run", run) + + binding = execution._provider_binding("claude") + + command = observed["command"] + assert command[1:4] == ("--setting-sources", "", "--settings") + assert json.loads(command[4])["env"] == execution.CLAUDE_FIRST_PARTY_ENV + assert command[5:] == ("auth", "status") + assert binding["auth_route"]["apiProvider"] == "firstParty" + + +def test_claude_run_settings_disable_third_party_routes(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + plugin_root = tmp_path / "plugin" + workspace.mkdir() + scratch.mkdir() + plugin_root.mkdir() + + settings = execution._claude_settings(workspace, scratch, plugin_root) + + for name, value in execution.CLAUDE_FIRST_PARTY_ENV.items(): + assert settings["env"][name] == value + + +def test_shakedown_requires_first_party_claude_route(monkeypatch): + def binding(provider): + if provider == "claude": + return { + "auth_route": { + "loggedIn": True, + "authMethod": "third_party", + "apiProvider": "bedrock", + } + } + return {"requested_model": execution.EXPECTED_MODELS[provider]} + + monkeypatch.setattr(execution, "_provider_binding", binding) + + with pytest.raises(execution.ExecutionError, match="first-party route"): + execution._validated_provider_bindings() diff --git a/tests/test_agent_qualification_treatments.py b/tests/test_agent_qualification_treatments.py new file mode 100644 index 0000000..78ca1f6 --- /dev/null +++ b/tests/test_agent_qualification_treatments.py @@ -0,0 +1,343 @@ +import hashlib +import json +import random +import subprocess +import sys +from pathlib import Path + +import pytest + +from experiments import agent_qualification +from experiments import agent_qualification_treatments as treatments + +BASELINE_RUNNER_SHA256 = "6ccf1f0ceee34672e049c428cb368d98a1c36270aeff21900f25b79e659261ef" + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _synthetic_manifest(): + return { + "cases": [ + { + "id": case_id, + "status": "ready", + "snapshot_id": f"sha256:{case_id}", + "revision": f"revision-{case_id}", + "commands": "native-test", + "declared_claim": "price-time matching", + "excluded_scope": "production readiness", + } + for case_id in ("case-alpha", "case-beta") + ] + } + + +def test_frozen_baseline_runner_is_byte_identical(): + assert _sha256(agent_qualification.RUNNER_PATH) == BASELINE_RUNNER_SHA256 + + +def test_treatment_prompt_is_identical_between_docs_and_skill(): + manifest = _synthetic_manifest() + for case in manifest["cases"]: + if case["status"] != "ready": + continue + rendered = treatments.render_prompt(case) + assert rendered == agent_qualification.render_prompt(case, "docs") + assert rendered == agent_qualification.render_prompt(case, "skill") + assert "tracebook-conformance==0.6.0" in rendered + + +def test_master_plan_is_two_matched_blocks_in_baseline_order(): + manifest = _synthetic_manifest() + entries = treatments._master_entries(manifest) + baseline_cells = [ + { + "case_id": case["id"], + "agent": agent, + "repetition": repetition, + } + for case in manifest["cases"] + for agent in ("codex", "claude") + for repetition in range(1, treatments.REPETITIONS + 1) + ] + random.Random(agent_qualification.RANDOMIZATION_SEED).shuffle(baseline_cells) + + assert len(entries) == 24 + assert [entry["condition"] for entry in entries[:12]] == ["docs"] * 12 + assert [entry["condition"] for entry in entries[12:]] == ["skill"] * 12 + for offset in (0, 12): + assert [ + { + "case_id": entry["case_id"], + "agent": entry["agent"], + "repetition": entry["repetition"], + } + for entry in entries[offset : offset + 12] + ] == baseline_cells + assert len({entry["run_id"] for entry in entries}) == 24 + assert len({entry["evidence_id"] for entry in entries}) == 24 + + +def test_skill_is_candidate_agnostic_and_exactly_one_measured_file(tmp_path, monkeypatch): + monkeypatch.setattr(treatments, "PRIVATE_ROOT", tmp_path) + skill_path = tmp_path / "skill" / "SKILL.md" + skill_path.parent.mkdir() + skill_path.write_text( + "---\n" + f"name: {treatments.SKILL_NAME}\n" + "description: Candidate-agnostic qualification workflow.\n" + "---\n\n" + "Inspect the supplied candidate and preserve identity lifecycles.\n" + ) + skill = treatments._load_skill(skill_path, _synthetic_manifest()) + + assert skill["name"] == treatments.SKILL_NAME + assert skill["sha256"] == _sha256(skill_path) + assert b"heldout" not in skill["content"].lower() + assert b"order-matcher" not in skill["content"].lower() + assert b"matching-engine-rs" not in skill["content"].lower() + + +def test_skill_leakage_audit_rejects_heldout_identifier(tmp_path, monkeypatch): + monkeypatch.setattr(treatments, "PRIVATE_ROOT", tmp_path) + skill = tmp_path / "SKILL.md" + skill.write_text( + "---\n" + f"name: {treatments.SKILL_NAME}\n" + "description: synthetic\n" + "---\n" + "Target c1-heldout directly.\n" + ) + manifest = {"cases": [{"id": "c1-heldout"}]} + + with pytest.raises(treatments.EvaluationError, match="leakage audit"): + treatments._load_skill(skill, manifest) + + +def test_private_output_rejects_escape_before_creating_parent(tmp_path, monkeypatch): + private_root = tmp_path / "private" + private_root.mkdir() + outside = tmp_path / "outside" / "result.json" + monkeypatch.setattr(treatments, "PRIVATE_ROOT", private_root) + + with pytest.raises(treatments.EvaluationError, match="must stay inside"): + treatments._private_output(outside, "test output") + + assert not outside.parent.exists() + + +def test_master_plan_shape_rejects_tampering(): + entries = [ + { + "ordinal": index, + "run_id": f"run-{index}", + "evidence_id": f"evidence-{index}", + "condition": "docs" if index <= 12 else "skill", + } + for index in range(1, 25) + ] + assert len(treatments._validate_plan_entry_shape(entries)) == 24 + entries[12]["evidence_id"] = entries[0]["evidence_id"] + + with pytest.raises(treatments.EvaluationError, match="evidence IDs"): + treatments._validate_plan_entry_shape(entries) + + +def test_claude_treatment_command_is_skill_capable_and_customization_isolated( + tmp_path, +): + command = treatments._claude_treatment_command(tmp_path / "settings.json", tmp_path / "plugin") + + assert "--safe-mode" not in command + assert "--disable-slash-commands" not in command + assert command[command.index("--setting-sources") + 1] == "" + assert command[command.index("--plugin-dir") + 1] == str(tmp_path / "plugin") + assert "Skill" in command[command.index("--tools") + 1].split(",") + assert command[command.index("--model") + 1] == "claude-opus-4-8" + assert command[command.index("--mcp-config") + 1] == '{"mcpServers":{}}' + + +def test_claude_wrapper_tree_contains_no_semantic_payload(tmp_path): + root = tmp_path / "claude-wrapper" + manifest = root / ".claude-plugin" / "plugin.json" + manifest.parent.mkdir(parents=True) + manifest.write_text('{"name":"agent-qualification-treatment","version":"1.0.0"}\n') + record = treatments._tree_record(root) + + assert record["file_count"] == 1 + assert record["files"][0]["path"] == ".claude-plugin/plugin.json" + + +def test_claude_settings_make_plugin_read_only_to_agent(tmp_path): + workspace = tmp_path / "workspace" + scratch = tmp_path / "scratch" + plugin = scratch / "plugin" + workspace.mkdir() + scratch.mkdir() + plugin.mkdir() + + settings = treatments._claude_treatment_settings(workspace, scratch, plugin) + + assert "CLAUDE_CONFIG_DIR" not in settings["env"] + assert "HOME" not in settings["env"] + assert settings["env"]["ZDOTDIR"] == str(scratch / "zsh-env") + assert (scratch / "zsh-env" / ".zshenv").read_text() == ( + f"export HOME={scratch / 'shell-home'}\n" + ) + assert str(plugin) in settings["sandbox"]["filesystem"]["denyWrite"] + assert f"Edit(/{plugin}/**)" in settings["permissions"]["deny"] + + +def test_codex_native_surface_differs_only_by_skill_file(tmp_path, monkeypatch): + auth = tmp_path / "auth.json" + auth.write_text("{}\n") + monkeypatch.setattr(agent_qualification, "CODEX_AUTH_PATH", auth) + workspace = tmp_path / "workspace" + workspace.mkdir() + skill = b"synthetic frozen skill\n" + + docs_root = tmp_path / "docs" + docs_scratch = docs_root / "scratch" + docs_root.mkdir() + docs_scratch.mkdir() + docs_environment: dict[str, str] = {} + docs_command, docs_surface, _ = treatments._prepare_native_surface( + agent="codex", + condition="docs", + workspace=workspace, + external_run_root=docs_root, + scratch=docs_scratch, + environment=docs_environment, + skill_content=skill, + ) + + skill_root = tmp_path / "skill" + skill_scratch = skill_root / "scratch" + skill_root.mkdir() + skill_scratch.mkdir() + skill_environment: dict[str, str] = {} + skill_command, skill_surface, _ = treatments._prepare_native_surface( + agent="codex", + condition="skill", + workspace=workspace, + external_run_root=skill_root, + scratch=skill_scratch, + environment=skill_environment, + skill_content=skill, + ) + + assert docs_surface["skill_installed"] is False + assert skill_surface["skill_installed"] is True + assert not (docs_root / "codex-home" / "skills" / treatments.SKILL_NAME).exists() + installed = skill_root / "codex-home" / "skills" / treatments.SKILL_NAME / "SKILL.md" + assert installed.read_bytes() == skill + for command, root in ((docs_command, docs_root), (skill_command, skill_root)): + permission = next(item for item in command if item.startswith("permissions.agent-eval=")) + assert f'"{root / "codex-home" / "skills"}"="read"' in permission + + +def test_native_surface_final_audit_detects_absence_and_drift(tmp_path): + skill = b"synthetic frozen skill\n" + external_root = tmp_path / "run" + scratch = external_root / "scratch" + scratch.mkdir(parents=True) + + docs = treatments._audit_native_surface( + agent="claude", + condition="docs", + external_run_root=external_root, + scratch=scratch, + skill_content=skill, + ) + assert docs["mutated"] is False + + path = scratch / "claude-plugin" / "skills" / treatments.SKILL_NAME / "SKILL.md" + path.parent.mkdir(parents=True) + path.write_bytes(b"drift") + treatment = treatments._audit_native_surface( + agent="claude", + condition="skill", + external_run_root=external_root, + scratch=scratch, + skill_content=skill, + ) + assert treatment["mutated"] is True + + +def test_transcript_terminal_state_requires_clean_completion(tmp_path): + codex = tmp_path / "codex.jsonl" + codex.write_text('{"type":"turn.completed"}\n') + claude = tmp_path / "claude.jsonl" + claude.write_text( + '{"type":"result","terminal_reason":"completed","subtype":"success",' '"result":"done"}\n' + ) + failed = tmp_path / "failed.jsonl" + failed.write_text('{"type":"error","message":"network"}\n') + + assert treatments._transcript_terminal_state(codex, "codex") == "completed" + assert treatments._transcript_terminal_state(claude, "claude") == "completed" + assert treatments._transcript_terminal_state(failed, "codex") == "failed" + + +def test_claude_catalog_audit_requires_only_wrapper_and_expected_skill(tmp_path): + transcript = tmp_path / "claude.jsonl" + transcript.write_text( + json.dumps( + { + "type": "system", + "subtype": "init", + "model": "claude-opus-4-8", + "plugins": [{"name": "agent-qualification-treatment"}], + "skills": [f"agent-qualification-treatment:{treatments.SKILL_NAME}"], + "mcp_servers": [], + "tools": [ + "Bash", + "Edit", + "Glob", + "Grep", + "Read", + "Skill", + "WebFetch", + "WebSearch", + "Write", + ], + } + ) + + "\n" + ) + + assert treatments._claude_catalog_audit(transcript, condition="skill")["valid"] is True + assert treatments._claude_catalog_audit(transcript, condition="docs")["valid"] is False + + +def test_measurement_policy_uses_conservative_paired_timing_gate(): + timing = treatments.MEASUREMENT_POLICY["timing_gate"] + + assert timing["minimum_matched_autonomous_safe_pass_pairs"] == 3 + assert timing["both_cases_required"] is True + assert timing["skill_safe_pass_rate_must_not_be_lower"] is True + assert timing["pass_threshold"] == 0.70 + + +def test_official_run_rejects_timeout_drift_before_freeze_validation(): + with pytest.raises(treatments.EvaluationError, match="timeout must remain"): + treatments.execute_run("not-a-run", timeout_seconds=1) + + +def test_direct_script_entrypoint_smoke(): + completed = subprocess.run( + [ + sys.executable, + str(treatments.RUNNER_PATH), + "--help", + ], + cwd=treatments.REPOSITORY_ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0 + assert "create the immutable 24-run plan" in completed.stdout