diff --git a/.serena/memories/morphic_control_plane_strategy.md b/.serena/memories/morphic_control_plane_strategy.md new file mode 100644 index 0000000..0898988 --- /dev/null +++ b/.serena/memories/morphic_control_plane_strategy.md @@ -0,0 +1,815 @@ +# Morphic Control Plane Strategy + +Decision date: 2026-07-14 +Last updated: 2026-07-20 + +## Product decision + +Morphic will not try to beat Claude Code, Codex CLI, or Gemini CLI by cloning +their terminal UX and reimplementing every native harness feature. Morphic will +be the multi-engine control plane that preserves and coordinates those native +agent runtimes. + +The winning category is: + +> One terminal and canonical workspace harness for routing, supervising, +> comparing, handing off, and auditing work across Claude Code, Codex CLI, +> Gemini CLI, OpenHands, Ollama, and direct LLM gateways. + +Native engines remain responsible for their strongest internal agent loops, +tools, subagents, skills, MCP integrations, and provider-specific behavior. +Morphic owns cross-engine session state, context projection, permission policy, +audit events, cost, normalized results, worktree isolation, evidence-based +selection, and handoff. + +## Current-state finding + +The Chat CLI foundation is strong and well tested, but it is still a technical +preview rather than a daily-driver coding agent: + +- `morphic chat` and `morphic code` default to `LocalChatCouncilRuntime`, which + returns deterministic planning text instead of running a coding agent. +- Chat hook and tool executors default to no-op. +- Route-backed council always invokes planner, critic, and leader sequentially, + adding latency and cost even when one engine is enough. +- External CLI drivers are one-shot subprocess adapters. They do not yet expose + streaming events, steering, approvals, resume, subagent state, or native + harness activity through Morphic. +- Skills, MCP, hooks, memory, and context features exist in the wider product, + but are not yet one coherent Chat CLI agent loop. + +Validation on 2026-07-14: 3,478 unit tests passed and Ruff was clean. + +## Implementation priorities + +1. Connect `morphic code` to a single real routed engine without forcing a + three-role council. Start behind explicit opt-in until permission propagation + and failure behavior are verified. +2. Define a normalized engine event stream: engine start, assistant delta, tool + request/result, approval, file change, verification, subagent activity, cost, + wait state, and completion. +3. Upgrade Claude Code, Codex CLI, and Gemini CLI adapters from one-shot result + capture to resumable, steerable native sessions while preserving their skills, + MCP, hooks, and permission behavior. +4. Use adaptive orchestration: one engine for simple work, critic on risk or + uncertainty, council only for complex or disputed work, worktrees for parallel + writers, and cross-engine handoff after failure. +5. Build `.morphic` harness inspect/diff/export projections for AGENTS.md, + CLAUDE.md, GEMINI.md, skills, hooks, MCP, and permissions. Never overwrite + tool-specific files silently. +6. Prove the advantage with real repository benchmarks against Claude Code and + Codex alone: completion rate, accepted patch rate, elapsed time, cost, human + interventions, recovery rate, and context-handoff fidelity. + +## First implementation slice + +Add a route-backed direct runtime for Chat CLI that makes exactly one +`RouteToEngineUseCase` call, supports an explicit preferred engine, records the +engine result through existing chat events, and reports route failures rather +than silently presenting the deterministic local response as success. + +Expose this first as an explicit CLI mode. Keep the current local deterministic +mode available as dry-run/fallback behavior until external-engine permission +mapping and live verification are complete. + +## Phase 27 update + +Codex CLI is the first permission-aware native direct adapter. The deprecated +`--full-auto` path was replaced with explicit `--sandbox` mapping and `--cd` +workspace scoping. Morphic `read-only`, `workspace-write`, and +`danger-full-access` map to the equivalent Codex sandbox. Morphic +`confirm-destructive` is deliberately rejected because `codex exec` is +non-interactive and cannot preserve an approval prompt channel. + +Codex `--json` output is JSONL, not a single JSON object. Morphic now normalizes +thread, turn, tool, file change, plan, assistant, completion, failure, and error +records into provider-independent engine events while retaining the raw payload. +Until equivalent permission/workspace mappings exist for other native engines, +Chat CLI direct route explicitly requires `--engine codex_cli`. + +## Phase 28 update + +Normalized native events are now durable chat state rather than metadata visible only +on the final engine result. Direct-runtime turns carry `AgentEngineEvent` values, and +the send-message use case appends each one to the session ledger before the matching +council argument and assistant response. Raw Codex JSONL payloads remain attached for +audit and future parser evolution. + +Workspace and permission controls now travel through `ScopedAgentEnginePort`, a narrow +capability separate from the common engine contract. Routing skips engines that do not +implement it instead of calling them with controls they might ignore. Codex is the first +scoped adapter. At the end of Phase 28, execution still buffered subprocess output until +completion; Phase 29 below closes that delivery gap. + +## Phase 29 update + +Codex direct execution now has a real incremental path. The subprocess runner drains +stdout and stderr concurrently; each JSONL stdout line is decoded with stateful thread +and sequence tracking and published before process completion. The application persists +the user message first, then appends native events immediately through its own ledger +sink. Buffered result metadata remains available for other callers without duplicating +streamed chat events. + +Streaming is expressed as narrow capabilities (`StreamingScopedAgentEnginePort` and +`StreamingCouncilRuntimePort`), so adapters that cannot honor live delivery are never +mistaken for adapters that can. The next gaps are terminal live rendering and Codex +thread resume/steering; the durability pipeline itself is now in place. + +## Phase 30 update + +The line-oriented terminal now surfaces selected native events as concise progress after +their durable ledger append. Rendering is allowlisted to run, tool, file, plan, +completion, and error state. Raw provider payloads, generic progress/reasoning, unknown +events, and assistant-message content stay out of the progress channel. Details are +whitespace-normalized and capped, and presentation failure cannot cancel execution or +erase audit history. + +The next control-plane primitive is native session continuity: resume the stored Codex +thread without losing Morphic workspace and permission guarantees, then expose steering +and cross-engine handoff on top of the same normalized ledger. + +## Phase 31 update + +Morphic now resumes an explicit Codex thread from its own append-only ledger. Native +session identity is stored with engine, original workspace root, and permission mode; +ledger replay reconstructs that binding after `morphic chat --resume`. The direct runtime +fails closed if current scope differs. Codex receives explicit sandbox/cwd options plus +`resume `; Morphic never relies on Codex's ambiguous global `--last`. + +Resume is a separate adapter capability rather than an assumption attached to all +streaming engines. The next gap is interactive steering/cancellation, followed by a +Claude Code adapter implementing the same scoped event and continuity contracts. + +## Phase 32 update + +Native process cancellation is now resource-safe. Both buffered and streaming subprocess +paths terminate the child when their asyncio task is cancelled, escalate to kill after a +two-second grace period, and re-raise the original cancellation. This closes the orphan +process risk before adding a user-facing `/cancel` or steering control channel. + +## Phase 33 update + +Claude Code is now a permission-aware scoped adapter. Morphic maps read-only to `plan`, +workspace-write to `acceptEdits`, and danger-full-access to an explicit bypass mode; it +rejects confirm-destructive in headless execution. The driver runs in the requested cwd. + +More importantly for the control-plane strategy, Morphic no longer forces Claude to +user-only settings or a hard-coded tool allowlist. Claude's project/local settings, +CLAUDE.md discovery, skills, hooks, MCP, plugins, and native tool policy remain intact. +The next slice is `stream-json` normalization and explicit session resume; only after +that should Chat CLI direct mode allow `claude_code` alongside `codex_cli`. + +## Phase 34 update + +Claude Code now implements the same normalized streaming and provenance-checked resume +contracts as Codex. Morphic maps Claude system init, assistant text/tool use, user tool +results, and final results into the shared event vocabulary while retaining raw payloads. +Explicit Claude session ids are resumed only inside their original workspace and +permission scope. + +Chat CLI direct mode now supports both `codex_cli` and `claude_code`. This is the first +working proof of the product thesis: two strong native harnesses keep their provider +features while Morphic owns a common durable ledger, progress view, safety boundary, +cost/result envelope, and session continuity. Next comes streaming input/steering and a +same-task comparative benchmark. + +## Phase 35 update + +Native resume identity is now provider-pinned end to end. A resume request carries its +owner engine as well as the session id; Morphic rejects preferred-engine mismatches and +skips every non-owner fallback before availability checks or process execution. Claude +session ids can never be offered to Codex, and Codex thread ids can never be offered to +Claude. This closes a subtle but critical cross-provider continuity failure introduced +when the second resumable native adapter came online. + +## Phase 36 update + +Streaming cancellation is now visible in Morphic's durable control plane, not only in +subprocess cleanup. The send-message use case appends a `turn_cancelled` event after the +user message and every native event received before interruption, then re-raises the +original cancellation. It does not manufacture a council decision or assistant success +for incomplete work. + +Both terminal entry points now report Ctrl-C as `Cancelled.` with exit code 130. Together +with Phase 32 child-process termination, cancellation now has end-to-end semantics from +the user's terminal through the application ledger to the provider process. The next +step is an explicit steering/cancellation control channel that can stop a running turn +without exiting the Morphic process. Phase 36 passed all 3,523 unit tests with +repository-wide Ruff clean. + +## Phase 37 update + +Interactive chat now has an addressable active-turn controller. While a turn is running, +Ctrl-C is routed to the child turn task rather than the parent REPL. The existing +cancellation chain still terminates the provider process and appends `turn_cancelled`, +but the REPL reloads its `ChatSession` from the durable ledger and accepts the next +prompt. Replaying first is essential: it prevents the in-memory pre-turn sequence from +colliding with user, engine, and cancellation events already appended during the +interrupted turn. + +The controller distinguishes its own cancellation request from cancellation of the +outer caller. An embedding application can still cancel the whole REPL with ordinary +asyncio semantics; only an explicit active-turn request becomes `TurnCancelledError`. +Repeated Ctrl-C requests do not interrupt cancellation cleanup. The controller also +restores the previous SIGINT handler when the turn ends, so idle Ctrl-C and one-shot +`morphic code` keep exit-code 130 behavior. User input and cancellation are durable for +non-streaming runtimes too, giving local council and native direct turns the same replay +contract. Phase 37 passed all 3,528 unit tests with repository-wide Ruff clean. + +The next control-plane slice is an addressable local/remote command transport over this +controller, followed by provider-specific steering input where the native CLI supports +it. Cancellation semantics no longer need to be reinvented by each transport. + +## Phase 38 update + +The active-turn controller is now addressable from another local terminal through an +explicitly enabled, authenticated loopback transport. `morphic chat --control` creates a +random-port server only while a turn is active. A session-scoped descriptor under +`.morphic/control/` carries protocol version, loopback address, port, and a random token; +the directory is mode 0700 and the descriptor is mode 0600. The descriptor is removed +when the turn completes or cancellation cleanup finishes. + +`morphic chat-control status` and `morphic chat-control cancel` provide the first +external control surface. The client refuses non-loopback descriptors, and the server +rejects invalid tokens, mismatched sessions, and unsupported commands before touching +the controller. The listener is opt-in and active-turn-only rather than a permanent +unauthenticated port. Remote use should initially go through an authenticated host +boundary such as SSH; direct network exposure remains deliberately unsupported. + +This is Morphic's first reusable remote-control primitive above provider processes. The +next slice is authenticated `steer`: cancel the current turn, queue a bounded replacement +prompt, replay the ledger, and continue the same provider-bound native session. Phase 38 +passed all 3,536 unit tests with repository-wide Ruff clean. + +## Phase 39 update + +The authenticated control transport now supports bounded `steer`. A steer request +accepts one non-empty replacement prompt of at most 2048 UTF-8 bytes. The first accepted +request queues the prompt and cancels the active child task; later requests during +cleanup cannot replace it. Invalid or oversized prompts are rejected before cancellation. + +After provider cleanup appends `turn_cancelled`, the REPL replays the ledger, appends a +`turn_steered` audit event, and submits the replacement as a normal `user_message`. This +restores the provider session id with its original workspace and permission provenance, +so Codex or Claude continues through the existing explicit native resume contract. +Remote prompts beginning with `/` are always provider messages and never become local +REPL slash commands. The prompt body is stored once in the normal user event; the steer +event records only source and UTF-8 byte length. + +This deliberately implements provider-neutral steering as cancel plus provenance-checked +resume, rather than depending on inconsistent provider stdin protocols. Phase 39 passed +all 3,542 unit tests with repository-wide Ruff clean. The next priority is a reproducible +same-task comparison harness for Codex alone, Claude alone, and Morphic-controlled runs, +with live paid execution remaining explicit opt-in. + +## Phase 40 update + +The first evidence harness is now offline and deterministic. One manifest pins the task, +workspace revision, verification checks, handoff assertions, and repetitions shared by +Codex CLI, Claude Code, and Morphic-controlled arms. A result set is accepted only when +every arm/trial cell appears exactly once and all passed assertions were declared before +the run. + +Reports compare completion, accepted patch, verification, median elapsed time, cost, +human interventions, recovery, and context-handoff fidelity. Verification and handoff +scores are derived from named assertions. Morphic deliberately does not collapse these +dimensions into a subjective weighted winner score; it reports leaders per metric and +emits timestamp-free, sorted-key JSON for reproducible review. + +`morphic benchmark agent-cli` only evaluates recorded JSON and cannot start Codex, +Claude, Morphic, or a paid API. The next slice should be a separate explicit opt-in +recorder with isolated worktrees, timeouts, cost caps, and captured verification evidence. +Phase 40 passed all 3,553 unit tests with repository-wide Ruff clean, and the built wheel +contains the benchmark package. + +## Phase 41 update + +The evidence harness now has a deliberately separate execution recorder. Its default +operation is a pure plan: validate manifest/config coverage, count trials, fingerprint +commands, and show the configured maximum estimate without creating a worktree or +starting an agent. + +Live recording requires three explicit signals: execute, acknowledgement that commands +may be paid, and a cost cap covering the complete configured estimate. Every arm/trial +runs at the pinned revision in its own detached worktree outside the source repository. +Commands receive argv directly without a shell, have bounded timeouts, and worktrees are +released through `finally` cleanup. + +Persisted evidence contains hashes and byte counts for argv/stdout/stderr plus exit, +timeout, elapsed, verification, and handoff outcomes. It never contains raw task prompts +or command output and never overwrites an existing evidence file. The authorized estimate +cap is audit data, not a claim that provider billing can be hard-stopped mid-request. +Actual provider cost, human interventions, recovery classification, and accepted-patch +review remain `pending_adjudication`; Morphic will not manufacture those values from +process success. The next slice is receipt parsing and deterministic adjudication into +the Phase 40 observation schema before any paid benchmark campaign. +Phase 41 passed all 3,566 unit tests with repository-wide Ruff clean. A real temporary +Git repository test verified pinned detached-worktree creation and removal, and the built +wheel contains both comparison and recorder modules. Exclusive evidence publication was +also verified to preserve an existing file under a competing write. + +## Phase 42 update + +Provider cost and human review now have an explicit evidence join instead of being +hand-entered directly into comparison observations. While raw agent stdout exists in +memory, the recorder attempts to normalize a privacy-safe receipt and then discards the +raw content. Codex usage is priced through the existing deterministic calculator using a +model or configured model hint; Claude retains its provider-reported total; Morphic uses +a strict `morphic_benchmark_receipt` envelope. + +Receipts enforce provider-specific cost sources, non-negative usage, recalculated Codex +cost, and zero parse errors. A campaign becomes `normalized_receipts` only when every +trial parsed successfully. Missing data never silently becomes zero cost. + +Independent review decisions bind accepted patch, human interventions, and recovery to +the exact agent argv SHA-256 and a review artifact SHA-256. The offline finalizer joins +those reviews with machine evidence, recomputes verification/handoff outcomes, enforces +the complete trial matrix and authorized actual-cost total, and rejects acceptance of a +failed provider/process run. It then emits the exact Phase 40 result schema with stable +JSON ordering and no timestamp. + +No paid campaign was executed. The remaining pre-campaign gap is first-party Morphic +receipt emission and committed configuration/review templates, followed by a zero-cost +local rehearsal. +Phase 42 passed all 3,582 unit tests with repository-wide Ruff clean, and the built wheel +contains the comparison, recorder, receipt, and adjudication modules. + +## Phase 43 update + +Morphic now emits its own canonical benchmark receipt instead of requiring a wrapper to +manufacture one. `morphic code --benchmark-receipt` preserves the ordinary human output +and appends one sorted JSON envelope as the final stdout line. The envelope aggregates +non-negative token counters found in normalized completion events, sums council-turn +cost, and identifies the participating Morphic engine set. Raw provider payloads never +enter the receipt. + +Failure and cancellation do not manufacture a zero-dollar receipt. Their final provider +cost may be unknown after an interrupted call, so receipt absence deliberately keeps the +campaign out of `normalized_receipts` until truthful evidence is available. + +The benchmark pipeline also has a committed zero-cost rehearsal contract. Its three +arms are internal Python fixtures with a hard configured estimate and actual total of +$0. They exercise detached worktrees, every provider receipt parser, hashed evidence, +review fingerprints, and deterministic finalization without starting Codex, Claude, +Morphic routing, or an API. Rehearsal review decisions intentionally leave +`accepted_patch=false`; synthetic success is not represented as patch quality. + +The real-repository rehearsal completed all three cells, published the five-file +manifest/config/evidence/reviews/results bundle exclusively, retained no raw command +output, and left no detached worktree behind. This closes the implementation gap before +a real campaign, but it is not competitive evidence. The next slice should add campaign +preflight and review-template authoring for a user-selected task/revision before any +paid execution is authorized. + +Phase 43 passed all 3,591 unit tests with repository-wide Ruff clean. The built +wheel contains the rehearsal module and both committed JSON templates. + +## Phase 43 publication checkpoint (2026-07-20) + +Phase 43 was committed as `5678d43` (`Rehearse agent CLI receipts locally`) and +pushed to `agent/codex-direct-stream-resume`. Draft PR #44 recognized that commit as +its head, and lint, unit tests, UI build, Docker build, GitGuardian, and the draft review +gate all passed. The local source branch and origin are synchronized. + +No paid benchmark campaign was executed. The next authorized development slice is +Phase 44: validate a user-selected manifest before execution, fingerprint required CLI +versions and commands, and generate an evidence-bound independent review template. +Campaign execution must remain a separate action requiring explicit paid acknowledgement +and a cost cap; preflight success alone never authorizes agent launch. + +## Phase 44 update + +Real campaigns now have a deterministic preflight artifact before execution. It requires +the manifest to contain the full resolved 40-character Git commit, validates exact +manifest/config coverage, normalizes operator-declared runtime versions, fingerprints +each version and every arm/check/handoff command, binds the complete manifest/config via +canonical SHA-256 without exposing the raw goal, and fixes +`execution_authorized=false`. Preflight resolves Git only; it never invokes a version +command, agent runtime, or paid API. + +After recording, Morphic can generate a complete review template with null human +decisions for every expected arm/trial. The template binds the exact preflight and +normalized evidence SHA-256 plus each expanded agent argv SHA-256. A reviewer must fill +every decision, attach a review artifact fingerprint, and change +`review_completed` to true. Finalization validates the evidence binding automatically +and, when `--preflight` is supplied, validates the preflight binding too. Legacy Phase 42 +reviews without these optional binding fields remain valid. + +A real zero-cost rehearsal on commit `88326ae` generated a three-arm non-authorizing +preflight and a three-decision null review template. No external agent, version probe, or +paid API was started. Phase 44 closes campaign-authoring ambiguity; it does not authorize +the first paid comparison. The next slice should validate reviewer separation and add a +read-only campaign status command before any paid run is considered. + +Phase 44 passed all 3,605 unit tests with repository-wide Ruff clean. The built +wheel contains the preflight module and runtime-version JSON template. + +## Phase 45 update + +Campaign review now has an explicit structural separation policy. A declaration names +the recorder/operator, the allowed reviewer IDs, and a minimum number of distinct +reviewers. Morphic normalizes and fingerprints that policy, binds it into pending and +completed review artifacts, rejects operator self-review, unauthorized IDs, insufficient +reviewer diversity, and policies impossible for the trial matrix. These IDs remain +operator declarations; the policy does not claim cryptographic identity authentication. + +`morphic benchmark agent-cli-status` is a read-only lifecycle validator spanning +`manifest_ready`, `preflight_ready`, `recorded`, `review_pending`, `review_complete`, and +`finalized`. It validates artifact ordering, manifest/preflight contract hashes, evidence +estimate and matrix, review/preflight/evidence/policy bindings, reviewer separation, and +recomputed final results. Every stage reports `paid_execution_authorized=false`; status +inspection cannot launch or authorize a campaign. + +A real zero-cost rehearsal at commit `d88f1f0` reached `review_pending` with a two-reviewer +policy. SHA-256 checks before and after status were identical for manifest, preflight, +evidence, review template, and policy files. No external agent, version probe, or paid API +was started. The next slice should add authenticated reviewer attestations or signed +artifact support without conflating declared IDs with verified human identity. + +Phase 45 passed all 3,621 unit tests with repository-wide Ruff clean. The built wheel +contains the campaign status, reviewer policy, and policy template artifacts. + +## Phase 46 update + +Reviewer separation now has cryptographic provenance without moving private keys into +Morphic. A self-fingerprinted trust declaration binds the exact Phase 45 review policy to +reviewer IDs, globally unique key IDs, Ed25519 public keys, public-key fingerprints, and +active/revoked status. New trust declarations require at least one active key for every +allowed reviewer; retained revoked keys support explicit rotation history and fail closed +if used for a new signature. + +`morphic benchmark agent-cli-attestation-template` produces one deterministic signing +request per distinct reviewer. Each statement binds the immutable task revision, +preflight and evidence hashes, policy and trust hashes, the completed reviews artifact, +and the canonical subset of decisions owned by that reviewer. The output contains the +exact base64 signing payload and no private-key material. Reviewers sign outside Morphic. + +Trust-bound finalization and read-only status require a complete detached-signature bundle. +One valid active-key Ed25519 signature is required for every distinct reviewer. Unknown or +revoked keys, invalid signatures, missing reviewer coverage, modified statements, and +reviews/policy/trust mixing are rejected. Status exposes `review_attestation_pending` +until verification succeeds, while unsigned Phase 42-45 campaigns keep their existing +backward-compatible path. Every status remains non-authorizing for paid execution. + +The important remaining boundary is key enrollment: a valid signature proves possession +of a key already placed in the trust declaration, but does not by itself prove real-world +identity or that the operator did not enroll a key they control. The next slice should +anchor reviewer keys in an organization CA or OIDC/Sigstore identity and sign the complete +campaign/result envelope, while retaining offline verification. + +Phase 46 passed all 3,633 unit tests with repository-wide Ruff clean. The built wheel +contains the attestation module and reviewer trust template and directly declares +`cryptography>=46.0.5`. No external agent, private-key file, version probe, or paid API +was started. + +## Phase 47 update + +Reviewer key enrollment can now be anchored outside the recorder/operator. A normalized, +self-fingerprinted offline Ed25519 organization authority is bound into anchored reviewer +trust. `agent-cli-reviewer-enrollment-template` produces a canonical signing payload for +every retained reviewer key without exposing or reading the authority private key. Each +authority certificate binds the authority, benchmark, review policy, exact reviewer trust, +reviewer/key IDs, and reviewer public-key fingerprint. Missing, duplicate, mixed, or +invalid certificates fail closed, and anchored finalization requires the complete bundle. + +The final campaign now has an optional authority-sealed boundary as well. +`agent-cli-campaign-envelope-template` binds manifest, preflight, evidence, completed +reviews, review policy, reviewer trust, authority enrollments, reviewer attestations, +results, and immutable campaign identity into one deterministic payload. It explicitly +fixes `paid_execution_authorized=false`. An authority-bound campaign remains at +`campaign_envelope_pending` until the external Ed25519 signature verifies; before key +enrollment it reports `reviewer_enrollment_pending`. Unanchored Phase 46 and unsigned +legacy campaigns retain their existing behavior. + +This closes the operator-controlled key-enrollment gap when an organization distributes +the authority root independently. The remaining trust-distribution gap is explicit: +Morphic does not yet prove safe delivery of the root public key, certificate expiry, +authority revocation/rotation, or append-only transparency inclusion. The next slice +should add versioned root rotation and revocation plus a transparency proof, and only then +map the same contracts onto OIDC/Sigstore if online identity is desired. + +Phase 47 passed all 3,646 unit tests with repository-wide Ruff clean. Real Ed25519 +enrollment, reviewer attestation, and final campaign envelope signatures ran with +in-memory keys only. The built wheel contains the authority module plus authority and +anchored-trust templates. No external authority, agent, network identity provider, +private-key file, version probe, or paid API was started. + +## Phase 48 update + +The offline organization trust anchor now has a recoverable history instead of being one +permanent key. Every root after the genesis carries an exact rotation statement signed by +its predecessor. The active root signs a self-fingerprinted ledger containing the ordered +generation chain and revocations. Verification rejects gaps, reorderings, reused roots, +unknown revocations, invalid predecessor signatures, ledger tampering, and a revoked +active root. + +New reviewer trusts can bind the active authority and the exact root-ledger SHA-256. +Finalization and the complete campaign envelope enforce that binding, while Phase 47 +artifacts omit the new optional field from their fingerprints and signing bytes. This is +important for Morphic's control-plane strategy: governance artifacts remain portable and +offline-verifiable instead of depending on one hosted service or one agent vendor. + +The campaign envelope can now be published into an append-only Merkle log. Leaves and +nodes use RFC 6962-style domain separation, the active root signs each tree head, and an +audit path proves inclusion of the exact campaign-envelope fingerprint. Ledger-bound +campaigns remain `transparency_pending` until that proof verifies. Complete old/new log +artifacts are accepted as append-only only when the old entries are an exact prefix. + +Phase 48 passed 3,655 unit tests with repository-wide Ruff clean using in-memory Ed25519 +keys only. No external authority, transparency server, identity provider, agent, or paid +API ran. Remaining boundaries are deliberate: genesis distribution and compromise reset +remain out-of-band, and complete-log prefix verification is not a compact consistency +proof or gossip protocol. The next trust slice should add compact consistency checkpoints +plus witnesses/gossip, or map the same verifier onto OIDC/Sigstore identities. + +## Phase 49 update + +Transparency growth can now be verified without exchanging complete historical logs. +Morphic implements the RFC 6962 `SUBPROOF` recursion and emits the unique minimal +consistency path between two active-root-signed tree heads. The verifier reconstructs +both advertised roots from the compact node list and rejects mismatched sizes, log IDs, +root ledgers, tree-head fingerprints or signatures, paths, and roots. The RFC seven-leaf +3-to-7 example shape and every prior size for trees through twelve leaves are covered. + +An optional witness layer reduces reliance on one log authority. Witness trust binds a +log ID, globally unique Ed25519 keys, active/revoked state, and a strict-majority quorum. +The majority rule guarantees that any two accepted quorums intersect. Detached witness +signatures cover the exact old/new roots and sizes, both tree-head fingerprints, authority +root ledger, compact proof, and witness trust. Missing quorum, duplicate witnesses, +unknown or revoked keys, invalid signatures, and same-size different-root checkpoints +fail closed. + +Campaign status preserves the Phase 48 inclusion-only path. When witness trust is +explicitly supplied, a campaign remains `witness_pending` until the compact proof and +witness checkpoint verify. Private-key-free consistency, witness-trust, and checkpoint +template CLI paths keep this governance portable across agent vendors and offline +environments. + +Phase 49 passed 3,664 unit tests with repository-wide Ruff clean using in-memory keys +only. No external log, witness, identity provider, agent, or paid API ran. The remaining +boundary is operational exchange: Morphic verifies witness artifacts but does not yet run +a gossip network, attest real-world witness identity, or maintain a durable global +checkpoint registry. The next slice should add an append-only local checkpoint store and +authenticated peer exchange before considering an online witness service. + +## Phase 50 update + +Witnessed checkpoints now have a durable local trust boundary. Each registry record binds +its registry ID, contiguous sequence, previous-record SHA-256, authority-root ledger, +witness trust, compact consistency proof, and signed checkpoint into a deterministic +self-fingerprint. Replay revalidates the entire hash chain plus every authority, Merkle, +and witness signature. A partial tail, sequence gap, altered fingerprint or link, stale +extension, and same-size different-root split view all fail closed. + +Append uses an exclusive file lock, `O_APPEND`, `fsync`, regular-file enforcement, and +mode 0600. Duplicate local appends and authenticated packet retries are idempotent, while +concurrent duplicate writers converge on one record. Read-only status does not create a +missing registry. + +Peer exchange remains transport-neutral and private-key-free. A self-fingerprinted peer +trust artifact supports globally unique Ed25519 key IDs plus active/revoked rotation, with +at least one active key per declared peer. Detached signatures bind the source peer and +exact registry record, checkpoint, log root, and tree size. Import authenticates the peer +before entering the same locked append and conflict checks used for local storage. + +Phase 50 passed 3,679 unit tests with repository-wide Ruff clean. Tests used in-memory +keys and local temporary files only; no external log, witness, peer service, agent, or paid +API ran. The remaining boundary is transport and catch-up: there is no listener, discovery, +real-world peer identity attestation, global consensus, or atomic multi-record range sync. +The next slice should add signed range bundles, atomic contiguous import, durable peer +cursors, and acknowledgements before exposing an online gossip transport. + +## Phase 51 update + +Offline peer catch-up now scales beyond one-record packets. A bounded contiguous range +statement binds the base hash, first/last sequence and record hashes, every record +fingerprint, registry, source peer, and peer trust to one Ed25519 signature. The receiver +authenticates that signature and validates every authority, witness, Merkle, record, and +range-chain binding before entering its write lock. + +Inside the lock, existing overlap must match exactly and only the missing contiguous +suffix is appended. Gaps and forks cause no mutation. The suffix is encoded as one batch, +and process-level write errors truncate back to the original size before the lock is +released; crash-created partial tails remain detectable and fail closed on replay. + +The receiver can sign an acknowledgement of the exact range and applied registry head. +The source persists verified acknowledgements in a separate mode-0600, locked, +hash-chained cursor ledger. Per source/receiver pair, cursor positions only advance; +exact retries are idempotent, while regression, same-sequence conflicts, invalid +signatures, and ledger tampering fail closed. + +Phase 51 passed 3,686 unit tests with repository-wide Ruff clean. All signatures used +in-memory keys and all persistence used temporary local files; no agent, network peer, +listener, or paid API ran. The remaining trust boundary precedes transport: range and ack +artifacts bind one peer-trust snapshot, so historical cursor replay after trust rotation +requires retaining that old artifact. The next slice should add a signed peer-trust +generation ledger and rollover continuity before any online gossip listener. + +## Phase 52 update + +Peer identity continuity is now versioned instead of depending on one current trust file. +Generation one remains an out-of-band genesis. Every successor trust carries an exact +rotation statement approved by a strict majority of distinct active peers from the +immediately preceding trust. The statement binds registry, generation, predecessor and +successor fingerprints, and the automatically computed majority threshold. + +Ledger replay verifies contiguous generations, stable registry identity, trust non-reuse, +certificate fingerprints, active predecessor keys, distinct peer identities, quorum, and +every Ed25519 signature. Minority approval, successor-only or revoked keys, invalid +signatures, gaps, reordering, reuse, and tampering fail closed. + +The ledger resolves an acknowledgement's historical peer-trust fingerprint. Cursor +storage and replay can therefore verify pre-rotation acknowledgements under the old trust +and append post-rotation acknowledgements under the successor trust without splitting the +durable cursor chain. CLI paths remain private-key-free and accept exactly one trust +snapshot or generation ledger. + +Phase 52 passed 3,691 unit tests with repository-wide Ruff clean. All rollover and +acknowledgement signatures used in-memory keys; no external peer, agent, listener, or paid +API ran. Remaining boundaries are genesis delivery, out-of-band pinning of the newest +ledger fingerprint, stale-ledger rollback detection, and transport. The next slice should +add an explicit opt-in loopback gossip transport with protocol versioning, nonce-based +challenge/replay defense, request limits, and deterministic shutdown before remote TLS. + +## Phase 53 update + +Checkpoint gossip now has a real but deliberately local network boundary. An operator must +explicitly start a short-lived server, which binds only to `127.0.0.1` on a random port. A +mode 0600 descriptor under an operator-selected path carries protocol version, registry, +source peer, a random instance id, and a 32-byte bearer token; its parent is mode 0700 and an +existing descriptor is never replaced. + +Every connection begins with a one-use 32-byte client nonce and a fresh server nonce. Both +the challenge response and the request/response pair use HMAC-SHA256, binding protocol, +instance, registry, source peer, operation, and payload. Reused nonces, wrong tokens, +endpoint/version mismatch, over-limit messages, exhausted capacity, and timeouts fail +closed. Limits are fixed at 64 KiB requests, 2 MiB responses, eight concurrent clients, +1,024 retained nonces/requests, and two seconds per read or dispatch. Max requests, bounded +listener lifetime, cancellation, or context exit close the listener plus active client +writers/tasks and remove only the owned descriptor. + +Transport authentication does not replace artifact trust. The server verifies every +pre-signed exact range before listening and never reads a private key. Fetch clients verify +the peer Ed25519 signature again using either a trust snapshot or the generation ledger; +full authority/witness/Merkle/registry verification still occurs in the existing atomic +range importer. Submitted acknowledgements are revalidated and pass through the durable +monotonic hash-chained cursor store. Invalid acknowledgement traffic cannot mutate it. + +Phase 53 added explicit serve/status/fetch/ack CLI paths and passed 3,697 unit tests with +repository-wide Ruff and focused mypy clean. Tests used loopback sockets and in-memory test +keys only; no external peer, agent, paid API, or non-loopback listener ran. The remaining +boundary is a bounded resumable catch-up loop, durable transport audit, newest trust-ledger +rollback pinning, peer discovery, and remote mTLS. Phase 54 should build the pull/catch-up +loop and rollback pin before any remote bind. + +## Phase 54 update + +Checkpoint gossip now has a bounded resumable pull loop instead of requiring an operator to +manually repeat status, fetch, and import. It starts from the fully verified local registry +count, selects a pre-signed range containing that exact next sequence, independently checks +the range signature and status fingerprint, and then uses the existing authority, witness, +Merkle consistency, overlap, and atomic registry import path. A bundle may include an +already-present exact prefix; only its missing suffix counts against the record budget. + +A separate mode 0600 JSONL sync audit holds a non-blocking exclusive process lock for the +entire loop. Records are deterministic, self-fingerprinted, hash-chained, fsynced, and bind +the exact loop-policy fingerprint. They capture imported range fingerprints, safe retry operation/attempt metadata, verified +registry-ahead recovery, trust advancement, and explicit stop reasons. Tokens, timestamps, +and raw exception text are never recorded. The registry remains the import source of truth: +if a crash occurs after registry fsync but before audit fsync, the last audited historical +head must still match the verified registry before a recovered record advances to its +current head. + +Every audit record pins the accepted peer-trust generation, trust fingerprint, and ledger +fingerprint. A later run rejects a ledger older than any pin and rejects a fork at a pinned +generation before opening the descriptor. A correctly signed contiguous ledger extension +advances the pin. This is rollback protection against stale or forked remote input under the +preserved local files, not protection from a local attacker capable of rewriting the entire +registry and audit history. + +Loop policy bounds rounds, newly imported records, attempts per request, and deterministic +backoff. Results stop with `up_to_date`, `range_gap`, `record_budget_exhausted`, +`round_budget_exhausted`, or `retry_exhausted`. The new gossip-sync CLI remains +private-key-free and passed 3,698 unit tests with repository-wide Ruff and focused mypy +clean. Only loopback sockets and in-memory test keys ran; no external peer, agent, paid API, +or non-loopback listener was used. + +Phase 55 should add an explicit remote mTLS boundary: peer-id-bound certificate/SPKI +enrollment anchored to the existing Ed25519 peer trust, TLS 1.3 only, address allowlists, +hostname/IP verification, certificate rotation continuity, and no plaintext fallback. + +## Phase 55 update + +Checkpoint gossip now has an explicit remote-capable mutual TLS boundary. A deterministic +enrollment statement binds one non-CA leaf certificate's normalized DER and SPKI SHA-256 +pins, subject and issuer, serial, validity interval, DNS/IP SANs, and dual client/server +authentication EKUs to one active Ed25519 peer identity signature. The signing template +and finalization CUI never read an identity private key. + +Certificate rotation is a per-peer contiguous generation chain. Each successor binds the +exact predecessor enrollment fingerprint. TLS trust construction reverifies every identity +signature under the exact peer-trust fingerprint, rejects missing generations or registry +changes, and exposes only the highest generation as active. Active certificate and SPKI pins +must also be unique across peers. Runtime server and client credentials must match both +active pins, so an enrolled but superseded certificate cannot authenticate. + +Transport protocol version two uses TLS 1.3 only and requires certificates on both sides. +The client verifies the CA chain, certificate hostname, explicit descriptor/server IP +allowlist, connected peer address, exact TLS-trust fingerprint, and server DER/SPKI pins. +The server verifies an explicit client IP allowlist, resolves the handshake leaf to exactly +one active peer, and requires the request's client peer ID to match. Its mode 0600 descriptor +contains no bearer token, and plaintext input never reaches the application protocol. + +One-use nonce replay defense plus the existing 64 KiB request, 2 MiB response, concurrency, +timeout, retained-nonce, request-count, and deterministic cleanup bounds remain in force. +Private key files must deny group and other access. The CUI now covers enrollment template, +detached-signature finalization, trust publication, mTLS serve, and mTLS status; any loaded +TLS trust is cryptographically rechecked against its peer trust before network use. + +Phase 55 passed 3,700 unit tests with repository-wide Ruff, focused mypy, and wheel build +clean. Tests used loopback sockets, temporary local CA/leaf certificates, and in-memory +Ed25519 identity keys only. No external peer, agent, paid API, or non-loopback listener ran. +Remaining operator boundaries are authentic CA/genesis distribution, DNS/address operations, +certificate revocation status and key custody. Phase 56 should wire fetch, acknowledgement, and resumable +sync CUI paths to protocol v2, then add revocation/expiry policy and authenticated discovery +without reintroducing bearer tokens or plaintext fallback. + +## Phase 56 update + +Checkpoint artifact operations no longer depend directly on the Phase 53 loopback transport. +Status, signed range fetch, and signed acknowledgement submission accept one typed authenticated +request sender, while an omitted sender preserves protocol v1 and every existing caller. The +reusable protocol-v2 client retains the exact mTLS descriptor, client peer identity, TLS trust, +leaf/key/CA paths, hostname, address allowlist, and timeout configuration. + +This split deliberately leaves artifact trust outside transport trust. Every fetched range is +still verified under its exact peer-trust generation after receipt, and an acknowledgement +response must return the exact submitted signed artifact. The Phase 54 catch-up loop now uses +the sender for status and fetch only; whole-loop locking, hash-chained audit, registry-ahead +recovery, trust rollback/fork pins, retry delays, round/record budgets, and stop reasons remain +one implementation rather than an mTLS fork. + +The CUI now exposes mTLS fetch, acknowledgement, and resumable sync. Before connecting, it +rebuilds TLS enrollment trust against the supplied peer-trust snapshot or the active generation +of a signed peer-trust ledger. Protocol v2 continues to require TLS 1.3, mutual certificates, +hostname and address checks, active DER/SPKI pins, one-use nonces, bounded messages, and a +token-free descriptor with no plaintext fallback. + +Phase 56 passed 3,700 unit tests with repository-wide Ruff, focused mypy, and wheel build +clean. No external peer, paid API, or non-loopback listener ran. Phase 57 should add a +peer-signed explicit certificate revocation chain and deterministic expiry/pre-expiry policy +before authenticated discovery. TLS handshake expiry remains enforced, but Morphic does not yet +provide revocation status, warning windows, OCSP/CRL policy, or automatic enrollment/ack signing. + +## Publication checkpoint (2026-07-15) + +Phases 26-31 form the first complete native Codex control-plane vertical slice: +single-engine direct routing, explicit workspace/permission mapping, provider-neutral +events, incremental durable streaming, safe terminal progress, and provenance-checked +thread resume. The implementation passed 3,513 unit tests with repository-wide Ruff +clean before publication. The next development sequence is cancellation/steering first, +then a Claude Code adapter behind the same scoped streaming and resume capabilities. + +## Phase 57 update + +TLS trust now carries an explicit peer-signed revocation chain. Revocations bind registry, +peer, enrollment generation/fingerprint, peer-trust fingerprint, reason, and UTC timestamp; +only an active peer identity key may sign them, and revoked generations are excluded from +active certificate/SPKI resolution. Trust fingerprints include the normalized revocation set, +and re-verification checks both enrollment and revocation signatures. + +Authenticated TLS server/client construction now applies a deterministic expiry policy before +opening transport: expired active leaves are rejected, while a configurable warning window is +reported as immutable `(peer_id, generation, expires_at, seconds_remaining)` tuples. Existing +callers retain defaults and TLS 1.3 handshake checks remain unchanged. + +Phase 57 unit verification remains green at 3,700 tests; next work is to expose revocation +issuance/trust updates and expiry warnings through the CUI and add dedicated regression fixtures. + +## Phase 58 update + +The checkpoint TLS trust CLI now accepts an optional signed revocation bundle and publishes +the resulting revocation-aware trust fingerprint. This keeps revocation verification offline, +explicit, and compatible with the existing enrollment/trust artifact workflow. PR #44 CI is +green after the Phase 57 transport policy changes. + +## Phase 59 update + +Added private-key-free `CheckpointPeerTlsRevocationTemplate` generation and signed revocation +finalization. The template binds one enrolled peer/generation, reason, timestamp, trust hash, +and eligible active identity keys; finalization verifies the detached Ed25519 signature before +returning the self-fingerprinted revocation artifact. +## Phase 60 update + +The revocation workflow is now exposed through two offline CUI commands: a private-key-free +template request and detached peer-signature finalization. Existing evidence is never +overwritten and finalized output remains deterministic JSON. +## Phase 61 update + +TLS identity CLI regression now covers trust load, private-key-free revocation template, +detached peer signature finalization, and artifact round-trip validation. +## Completion checkpoint + +The mTLS status CUI now exposes `tls_expiry_warnings` in JSON and the human summary, making +pre-expiry policy observable during actual operation. The current vertical slice is ready for +manual CLI smoke testing. diff --git a/application/use_cases/execute_chat_hook.py b/application/use_cases/execute_chat_hook.py new file mode 100644 index 0000000..6211427 --- /dev/null +++ b/application/use_cases/execute_chat_hook.py @@ -0,0 +1,112 @@ +"""Execute validated chat hooks through a hook executor port.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from domain.entities.chat_event import ChatEvent, ChatEventType +from domain.entities.chat_session import ChatSession +from domain.entities.hook import ( + HookDefinition, + HookDiagnostic, + HookExecutionRequest, + HookExecutionResult, + HookType, +) +from domain.ports.chat_session_store import ChatSessionStorePort +from domain.ports.hook_executor import HookExecutorPort +from domain.ports.hook_registry import HookRegistryPort + + +@dataclass(frozen=True) +class ExecuteChatHookResult: + session: ChatSession + events: list[ChatEvent] + diagnostics: list[HookDiagnostic] + hook_results: list[HookExecutionResult] + + +class ExecuteChatHookUseCase: + """Execute hooks already accepted by workspace validation policy.""" + + def __init__( + self, + *, + session_store: ChatSessionStorePort, + hook_registry: HookRegistryPort, + hook_executor: HookExecutorPort, + ) -> None: + self._session_store = session_store + self._hook_registry = hook_registry + self._hook_executor = hook_executor + + async def execute( + self, + *, + session: ChatSession, + hook_type: HookType, + ) -> ExecuteChatHookResult: + diagnostics = self._hook_registry.validate() + failures = [diagnostic for diagnostic in diagnostics if diagnostic.status == "FAIL"] + if failures: + names = ", ".join(diagnostic.name for diagnostic in failures) + raise ValueError(f"Hook diagnostics failed: {names}") + + current = session + events: list[ChatEvent] = [] + hook_results: list[HookExecutionResult] = [] + + for hook in self._hook_registry.hooks_for(hook_type): + if not hook.enabled: + current, skipped_event = current.record_event( + ChatEventType.HOOK_EXECUTION_SKIPPED, + self._skipped_payload_for(hook), + ) + events.append(skipped_event) + continue + + request = HookExecutionRequest( + session_id=current.id, + hook_name=hook.name, + hook_type=hook.hook_type, + command=hook.command, + source_path=hook.source_path, + ) + current, requested_event = current.record_event( + ChatEventType.HOOK_EXECUTION_REQUESTED, + request.model_dump(mode="json"), + ) + events.append(requested_event) + + hook_result = await self._hook_executor.execute(request) + hook_results.append(hook_result) + current, completed_event = current.record_event( + ChatEventType.HOOK_EXECUTION_COMPLETED, + { + **hook_result.model_dump(mode="json"), + "hook_name": hook.name, + "hook_type": hook.hook_type.value, + "source_path": hook.source_path, + }, + ) + events.append(completed_event) + + for event in events: + await self._session_store.append_event(event) + + return ExecuteChatHookResult( + session=current, + events=events, + diagnostics=diagnostics, + hook_results=hook_results, + ) + + def _skipped_payload_for(self, hook: HookDefinition) -> dict[str, str]: + return { + "command": hook.command, + "hook_name": hook.name, + "hook_type": hook.hook_type.value, + "reason": "disabled", + "source_path": hook.source_path, + "status": "skipped", + } diff --git a/application/use_cases/execute_chat_tool.py b/application/use_cases/execute_chat_tool.py index e0fd8f8..e12631d 100644 --- a/application/use_cases/execute_chat_tool.py +++ b/application/use_cases/execute_chat_tool.py @@ -5,10 +5,11 @@ from dataclasses import dataclass from typing import Any +from application.use_cases.execute_chat_hook import ExecuteChatHookUseCase from application.use_cases.plan_chat_hooks import PlanChatHooksUseCase from domain.entities.chat_event import ChatEvent, ChatEventType from domain.entities.chat_session import ChatSession, PermissionMode -from domain.entities.hook import HookType +from domain.entities.hook import HookExecutionResult, HookType from domain.ports.chat_session_store import ChatSessionStorePort from domain.ports.tool_executor import ( ToolExecutionRequest, @@ -35,10 +36,12 @@ def __init__( session_store: ChatSessionStorePort, tool_executor: ToolExecutorPort, hook_planner: PlanChatHooksUseCase | None = None, + hook_runner: ExecuteChatHookUseCase | None = None, ) -> None: self._session_store = session_store self._tool_executor = tool_executor self._hook_planner = hook_planner + self._hook_runner = hook_runner async def execute( self, @@ -57,7 +60,15 @@ async def execute( current = session events: list[ChatEvent] = [] - if self._hook_planner is not None: + if self._hook_runner is not None: + hook_result = await self._hook_runner.execute( + session=current, + hook_type=HookType.PRE_TOOL, + ) + current = hook_result.session + events.extend(hook_result.events) + self._raise_if_hook_failed(hook_result.hook_results, HookType.PRE_TOOL) + elif self._hook_planner is not None: hook_result = await self._hook_planner.execute( session=current, hook_type=HookType.PRE_TOOL, @@ -105,7 +116,17 @@ async def execute( ) tool_events.append(verification_event) - if self._hook_planner is not None: + if self._hook_runner is not None: + for event in tool_events: + await self._session_store.append_event(event) + events.extend(tool_events) + hook_result = await self._hook_runner.execute( + session=current, + hook_type=HookType.POST_TOOL, + ) + current = hook_result.session + events.extend(hook_result.events) + elif self._hook_planner is not None: for event in tool_events: await self._session_store.append_event(event) events.extend(tool_events) @@ -136,3 +157,12 @@ def _blocked_by_read_only( if session.permission_mode is not PermissionMode.READ_ONLY: return False return risk_level > RiskLevel.SAFE or tool_name not in _READ_ONLY_TOOLS + + def _raise_if_hook_failed( + self, + hook_results: list[HookExecutionResult], + hook_type: HookType, + ) -> None: + failed = [result for result in hook_results if not result.success] + if failed: + raise RuntimeError(f"{hook_type.value} hook failed") diff --git a/application/use_cases/resume_chat_session.py b/application/use_cases/resume_chat_session.py index fe9ed23..8195741 100644 --- a/application/use_cases/resume_chat_session.py +++ b/application/use_cases/resume_chat_session.py @@ -40,15 +40,12 @@ async def execute(self, session_id: str) -> ResumeChatSessionResult: if isinstance(mode_value, str): permission_mode = PermissionMode(mode_value) - status = ChatSessionStatus.ACTIVE - if any(event.type is ChatEventType.SESSION_ENDED for event in events): - status = ChatSessionStatus.ENDED - session = ChatSession( id=resolved_session_id, goal=goal, permission_mode=permission_mode, - status=status, - next_sequence=max(event.sequence for event in events) + 1, + status=ChatSessionStatus.ACTIVE, ) + for event in events: + session = session.replay_event(event) return ResumeChatSessionResult(session=session, events=events) diff --git a/application/use_cases/route_to_engine.py b/application/use_cases/route_to_engine.py index f746db2..c949671 100644 --- a/application/use_cases/route_to_engine.py +++ b/application/use_cases/route_to_engine.py @@ -15,10 +15,19 @@ from dataclasses import dataclass from application.use_cases.run_council_debate import RunCouncilDebateUseCase +from domain.entities.chat_session import PermissionMode from domain.entities.cognitive import AgentAction, Decision from domain.entities.council import SubtaskBrief from domain.ports.agent_affinity_repository import AgentAffinityRepository -from domain.ports.agent_engine import AgentEngineCapabilities, AgentEnginePort, AgentEngineResult +from domain.ports.agent_engine import ( + AgentEngineCapabilities, + AgentEngineEventSinkPort, + AgentEnginePort, + AgentEngineResult, + ResumableStreamingScopedAgentEnginePort, + ScopedAgentEnginePort, + StreamingScopedAgentEnginePort, +) from domain.ports.context_adapter import ContextAdapterPort from domain.ports.engine_cost_recorder import EngineCostRecorderPort from domain.ports.shared_task_state_repository import SharedTaskStateRepository @@ -108,6 +117,11 @@ async def execute( timeout_seconds: float = 300.0, context: str | None = None, task_id: str | None = None, + workspace_root: str | None = None, + permission_mode: PermissionMode | None = None, + event_sink: AgentEngineEventSinkPort | None = None, + resume_session_id: str | None = None, + resume_engine: AgentEngineType | None = None, ) -> AgentEngineResult: """Route to best available engine and execute. @@ -122,6 +136,24 @@ async def execute( BUG-003: Every attempt is recorded as a FallbackAttempt for transparency. BUG-002: Successful engine costs are recorded via CostTracker. """ + if resume_session_id is not None and ( + event_sink is None + or workspace_root is None + or permission_mode is None + or resume_engine is None + ): + raise ValueError( + "native resume requires engine, streaming, workspace, and permission context" + ) + if resume_session_id is None and resume_engine is not None: + raise ValueError("resume engine requires a native session id") + if ( + resume_engine is not None + and preferred_engine is not None + and resume_engine is not preferred_engine + ): + raise ValueError("preferred engine must match native resume engine") + # Extract topic for affinity lookup topic = TopicExtractor.extract(task) @@ -142,6 +174,15 @@ async def execute( last_result: AgentEngineResult | None = None for engine_type in chain: + if resume_engine is not None and engine_type is not resume_engine: + attempts.append( + FallbackAttempt( + engine=engine_type.value, + attempted=False, + skip_reason="resume_engine_mismatch", + ) + ) + continue driver = self._drivers.get(engine_type) if driver is None: logger.debug("Engine %s not registered, skipping", engine_type.value) @@ -175,11 +216,74 @@ async def execute( start = time.monotonic() try: - result = await driver.run_task( - task=effective_task, - model=model, - timeout_seconds=timeout_seconds, - ) + if workspace_root is not None or permission_mode is not None: + if ( + not isinstance(driver, ScopedAgentEnginePort) + or workspace_root is None + or permission_mode is None + ): + attempts.append( + FallbackAttempt( + engine=engine_type.value, + attempted=False, + skip_reason="scoped_execution_unsupported", + ) + ) + continue + if event_sink is not None: + if resume_session_id is not None: + if not isinstance( + driver, ResumableStreamingScopedAgentEnginePort + ): + attempts.append( + FallbackAttempt( + engine=engine_type.value, + attempted=False, + skip_reason="native_resume_unsupported", + ) + ) + continue + result = await driver.resume_task_scoped_stream( + task=effective_task, + resume_session_id=resume_session_id, + workspace_root=workspace_root, + permission_mode=permission_mode, + event_sink=event_sink, + model=model, + timeout_seconds=timeout_seconds, + ) + elif not isinstance(driver, StreamingScopedAgentEnginePort): + attempts.append( + FallbackAttempt( + engine=engine_type.value, + attempted=False, + skip_reason="streaming_scoped_execution_unsupported", + ) + ) + continue + else: + result = await driver.run_task_scoped_stream( + task=effective_task, + workspace_root=workspace_root, + permission_mode=permission_mode, + event_sink=event_sink, + model=model, + timeout_seconds=timeout_seconds, + ) + else: + result = await driver.run_task_scoped( + task=effective_task, + workspace_root=workspace_root, + permission_mode=permission_mode, + model=model, + timeout_seconds=timeout_seconds, + ) + else: + result = await driver.run_task( + task=effective_task, + model=model, + timeout_seconds=timeout_seconds, + ) except Exception as exc: elapsed = time.monotonic() - start logger.warning( diff --git a/application/use_cases/send_chat_message.py b/application/use_cases/send_chat_message.py index bb493c3..6192712 100644 --- a/application/use_cases/send_chat_message.py +++ b/application/use_cases/send_chat_message.py @@ -2,19 +2,55 @@ from __future__ import annotations +import asyncio +import logging from dataclasses import dataclass +from domain.entities.agent_engine_event import AgentEngineEvent from domain.entities.chat_event import ChatEvent, ChatEventType from domain.entities.chat_session import ChatSession +from domain.entities.council_runtime import CouncilTurn from domain.entities.workspace_context import ContextIndex +from domain.ports.agent_engine import AgentEngineEventSinkPort from domain.ports.chat_session_store import ChatSessionStorePort -from domain.ports.council_runtime import CouncilRuntimePort +from domain.ports.council_runtime import CouncilRuntimePort, StreamingCouncilRuntimePort + +logger = logging.getLogger(__name__) + + +class _LedgerEngineEventSink(AgentEngineEventSinkPort): + def __init__( + self, + *, + session: ChatSession, + events: list[ChatEvent], + session_store: ChatSessionStorePort, + observer: AgentEngineEventSinkPort | None = None, + ) -> None: + self.session = session + self._events = events + self._session_store = session_store + self._observer = observer + + async def publish(self, engine_event: AgentEngineEvent) -> None: + self.session, event = self.session.record_event( + ChatEventType.ENGINE_EVENT, + engine_event.model_dump(mode="json"), + ) + self._events.append(event) + await self._session_store.append_event(event) + if self._observer is not None: + try: + await self._observer.publish(engine_event) + except Exception: + logger.warning("Native engine progress observer failed", exc_info=True) @dataclass(frozen=True) class SendChatMessageResult: session: ChatSession events: list[ChatEvent] + turns: tuple[CouncilTurn, ...] class SendChatMessageUseCase: @@ -23,9 +59,11 @@ def __init__( *, session_store: ChatSessionStorePort, council_runtime: CouncilRuntimePort, + engine_event_observer: AgentEngineEventSinkPort | None = None, ) -> None: self._session_store = session_store self._council_runtime = council_runtime + self._engine_event_observer = engine_event_observer async def execute( self, @@ -42,14 +80,47 @@ async def execute( {"text": message}, ) events = [user_event] - - turns, decision = await self._council_runtime.deliberate( - current, - context, - message, - ) + streaming = isinstance(self._council_runtime, StreamingCouncilRuntimePort) + await self._session_store.append_event(user_event) + persisted_count = 1 + if streaming: + sink = _LedgerEngineEventSink( + session=current, + events=events, + session_store=self._session_store, + observer=self._engine_event_observer, + ) + try: + turns, decision = await self._council_runtime.deliberate_stream( + current, + context, + message, + sink, + ) + except asyncio.CancelledError: + await self._append_cancellation(sink.session) + raise + current = sink.session + persisted_count = len(events) + else: + try: + turns, decision = await self._council_runtime.deliberate( + current, + context, + message, + ) + except asyncio.CancelledError: + await self._append_cancellation(current) + raise for turn in turns: + if not streaming: + for engine_event in turn.engine_events: + current, event = current.record_event( + ChatEventType.ENGINE_EVENT, + engine_event.model_dump(mode="json"), + ) + events.append(event) current, event = current.record_event( ChatEventType.COUNCIL_ARGUMENT, turn.model_dump(mode="json"), @@ -72,7 +143,14 @@ async def execute( ) events.append(assistant_event) - for event in events: + for event in events[persisted_count:]: await self._session_store.append_event(event) - return SendChatMessageResult(session=current, events=events) + return SendChatMessageResult(session=current, events=events, turns=tuple(turns)) + + async def _append_cancellation(self, session: ChatSession) -> None: + _, cancelled_event = session.record_event( + ChatEventType.TURN_CANCELLED, + {"reason": "caller_cancelled"}, + ) + await asyncio.shield(self._session_store.append_event(cancelled_event)) diff --git a/benchmarks/agent_cli_adjudication.py b/benchmarks/agent_cli_adjudication.py new file mode 100644 index 0000000..3dcfcd1 --- /dev/null +++ b/benchmarks/agent_cli_adjudication.py @@ -0,0 +1,347 @@ +"""Deterministically join recorder evidence and independent review decisions.""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_comparison import ( + SCHEMA_VERSION, + AgentCliArm, + AgentCliManifest, + RecordedResults, + TrialObservation, +) +from benchmarks.agent_cli_receipts import ProviderReceipt + +if TYPE_CHECKING: + from benchmarks.agent_cli_attestation import ReviewAttestationBundle, ReviewerTrust + from benchmarks.agent_cli_authority import BenchmarkAuthority, ReviewerEnrollmentBundle + from benchmarks.agent_cli_review_policy import ReviewerPolicy + from benchmarks.agent_cli_transparency import SignedAuthorityRootLedger + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +class HashedCommandEvidence(_FrozenModel): + argv_sha256: str = Field(pattern=_SHA256_PATTERN) + exit_code: int + timed_out: bool + elapsed_seconds: float = Field(ge=0.0) + stdout_sha256: str = Field(pattern=_SHA256_PATTERN) + stdout_bytes: int = Field(ge=0) + stderr_sha256: str = Field(pattern=_SHA256_PATTERN) + stderr_bytes: int = Field(ge=0) + + @property + def passed(self) -> bool: + return self.exit_code == 0 and not self.timed_out + + +class RecordedTrialEvidence(_FrozenModel): + arm: AgentCliArm + trial: int = Field(ge=1) + reserved_cost_usd: float = Field(ge=0.0) + agent: HashedCommandEvidence + checks: dict[str, HashedCommandEvidence] + handoff_assertions: dict[str, HashedCommandEvidence] + receipt: ProviderReceipt | None + completed: bool + passed_checks: tuple[str, ...] + passed_handoff_assertions: tuple[str, ...] + + @model_validator(mode="after") + def validate_derived_fields(self) -> RecordedTrialEvidence: + if self.completed != self.agent.passed: + raise ValueError("completed does not match agent command evidence") + if len(self.passed_checks) != len(set(self.passed_checks)): + raise ValueError("passed_checks must be unique") + calculated_checks = {name for name, evidence in self.checks.items() if evidence.passed} + if set(self.passed_checks) != calculated_checks: + raise ValueError("passed_checks do not match command evidence") + if len(self.passed_handoff_assertions) != len(set(self.passed_handoff_assertions)): + raise ValueError("passed_handoff_assertions must be unique") + calculated_handoffs = { + name for name, evidence in self.handoff_assertions.items() if evidence.passed + } + if set(self.passed_handoff_assertions) != calculated_handoffs: + raise ValueError("passed_handoff_assertions do not match command evidence") + return self + + +class RecordedEvidence(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + workspace_revision: str = Field(min_length=1) + estimated_max_cost_usd: float = Field(ge=0.0) + authorized_cost_cap_usd: float = Field(ge=0.0) + cost_collection: Literal["pending_adjudication", "normalized_receipts"] + trials: tuple[RecordedTrialEvidence, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_schema(self) -> RecordedEvidence: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + return self + + +class ReviewDecision(_FrozenModel): + arm: AgentCliArm + trial: int = Field(ge=1) + agent_argv_sha256: str = Field(pattern=_SHA256_PATTERN) + accepted_patch: bool + human_interventions: int = Field(ge=0) + recovery_attempted: bool + recovery_succeeded: bool + reviewer_id: str = Field(min_length=1) + review_artifact_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_recovery(self) -> ReviewDecision: + if self.recovery_succeeded and not self.recovery_attempted: + raise ValueError("recovery_succeeded requires recovery_attempted=true") + return self + + +class AdjudicationReviews(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + workspace_revision: str = Field(min_length=1) + preflight_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + evidence_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + review_policy_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + review_completed: Literal[True] | None = None + decisions: tuple[ReviewDecision, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_schema(self) -> AdjudicationReviews: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.reviewer_trust_sha256 is not None and self.review_policy_sha256 is None: + raise ValueError("reviewer trust binding requires a review policy binding") + return self + + +def recorded_evidence_sha256(evidence: RecordedEvidence) -> str: + """Fingerprint the complete normalized evidence artifact deterministically.""" + payload = json.dumps( + evidence.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _validate_identity( + manifest: AgentCliManifest, + evidence: RecordedEvidence, + reviews: AdjudicationReviews, +) -> None: + expected = (manifest.benchmark_id, manifest.task.id, manifest.task.workspace_revision) + if (evidence.benchmark_id, evidence.task_id, evidence.workspace_revision) != expected: + raise ValueError("evidence identity does not match manifest") + if (reviews.benchmark_id, reviews.task_id, reviews.workspace_revision) != expected: + raise ValueError("review identity does not match manifest") + + +def _index_complete_matrix( + rows: tuple[RecordedTrialEvidence, ...] | tuple[ReviewDecision, ...], + expected: set[tuple[AgentCliArm, int]], + *, + label: str, +) -> dict[tuple[AgentCliArm, int], RecordedTrialEvidence | ReviewDecision]: + indexed: dict[tuple[AgentCliArm, int], RecordedTrialEvidence | ReviewDecision] = {} + for row in rows: + key = (row.arm, row.trial) + if key in indexed: + raise ValueError(f"duplicate {label}: {row.arm.value}:{row.trial}") + indexed[key] = row + missing = expected - set(indexed) + extra = set(indexed) - expected + if missing: + details = ", ".join(f"{arm.value}:{trial}" for arm, trial in sorted(missing)) + raise ValueError(f"missing {label}: {details}") + if extra: + details = ", ".join(f"{arm.value}:{trial}" for arm, trial in sorted(extra)) + raise ValueError(f"unexpected {label}: {details}") + return indexed + + +def finalize_recorded_results( + manifest: AgentCliManifest, + evidence: RecordedEvidence, + reviews: AdjudicationReviews, + *, + review_policy: ReviewerPolicy | None = None, + reviewer_trust: ReviewerTrust | None = None, + attestations: ReviewAttestationBundle | None = None, + reviewer_authority: BenchmarkAuthority | None = None, + reviewer_enrollments: ReviewerEnrollmentBundle | None = None, + authority_root_ledger: SignedAuthorityRootLedger | None = None, +) -> RecordedResults: + """Create Phase 40 observations only after every evidence join validates.""" + attestation_inputs = (review_policy, reviewer_trust, attestations) + if reviews.reviewer_trust_sha256 is not None: + if any(value is None for value in attestation_inputs): + raise ValueError("trust-bound reviews require policy, trust, and attestations") + from benchmarks.agent_cli_attestation import verify_review_attestations + + assert review_policy is not None + assert reviewer_trust is not None + assert attestations is not None + verify_review_attestations( + review_policy, + reviewer_trust, + reviews, + attestations, + ) + authority_inputs = ( + reviewer_authority, + reviewer_enrollments, + authority_root_ledger, + ) + if reviewer_trust.reviewer_authority_sha256 is not None: + if reviewer_authority is None or reviewer_enrollments is None: + raise ValueError( + "authority-bound reviews require authority and reviewer enrollments" + ) + from benchmarks.agent_cli_authority import verify_reviewer_enrollments + + assert reviewer_authority is not None + assert reviewer_enrollments is not None + verify_reviewer_enrollments( + reviewer_authority, + review_policy, + reviewer_trust, + reviewer_enrollments, + ) + if reviewer_trust.authority_root_ledger_sha256 is not None: + if authority_root_ledger is None: + raise ValueError( + "ledger-bound reviews require an authority root ledger" + ) + from benchmarks.agent_cli_transparency import ( + verify_authority_root_ledger, + ) + + active_authority = verify_authority_root_ledger(authority_root_ledger) + if active_authority != reviewer_authority: + raise ValueError("reviewer authority is not the active authority root") + if ( + authority_root_ledger.statement.ledger_sha256 + != reviewer_trust.authority_root_ledger_sha256 + ): + raise ValueError( + "reviewer trust authority root ledger does not match ledger" + ) + elif authority_root_ledger is not None: + raise ValueError("authority root ledger requires ledger-bound trust") + elif any(value is not None for value in authority_inputs): + raise ValueError("reviewer enrollments require an authority-bound trust") + elif any( + value is not None + for value in ( + reviewer_trust, + attestations, + reviewer_authority, + reviewer_enrollments, + authority_root_ledger, + ) + ): + raise ValueError("attestations require a reviewer trust binding") + _validate_identity(manifest, evidence, reviews) + if ( + reviews.evidence_sha256 is not None + and reviews.evidence_sha256 != recorded_evidence_sha256(evidence) + ): + raise ValueError("review evidence fingerprint does not match evidence") + if evidence.cost_collection != "normalized_receipts": + raise ValueError("evidence cost_collection is not normalized_receipts") + expected = { + (arm, trial) + for arm in manifest.arms + for trial in range(1, manifest.repetitions + 1) + } + evidence_by_key = _index_complete_matrix(evidence.trials, expected, label="evidence") + reviews_by_key = _index_complete_matrix(reviews.decisions, expected, label="review") + + observations: list[TrialObservation] = [] + total_cost = 0.0 + for arm in manifest.arms: + for trial_number in range(1, manifest.repetitions + 1): + key = (arm, trial_number) + trial = evidence_by_key[key] + review = reviews_by_key[key] + assert isinstance(trial, RecordedTrialEvidence) + assert isinstance(review, ReviewDecision) + if set(trial.checks) != set(manifest.task.checks): + raise ValueError(f"check evidence mismatch for {arm.value}:{trial_number}") + if set(trial.handoff_assertions) != set(manifest.task.handoff_assertions): + raise ValueError(f"handoff evidence mismatch for {arm.value}:{trial_number}") + if trial.receipt is None: + raise ValueError(f"receipt missing for {arm.value}:{trial_number}") + if trial.receipt.provider is not arm: + raise ValueError(f"receipt provider mismatch for {arm.value}:{trial_number}") + if trial.receipt.parse_errors: + raise ValueError(f"receipt contains parse errors for {arm.value}:{trial_number}") + if review.agent_argv_sha256 != trial.agent.argv_sha256: + raise ValueError(f"review fingerprint mismatch for {arm.value}:{trial_number}") + + completed = trial.completed and trial.receipt.success + if review.accepted_patch and not completed: + raise ValueError( + f"accepted_patch requires completed trial for {arm.value}:{trial_number}" + ) + if review.recovery_succeeded and not completed: + raise ValueError( + f"recovery_succeeded requires completed trial for {arm.value}:{trial_number}" + ) + total_cost += trial.receipt.cost_usd + observations.append( + TrialObservation( + arm=arm, + trial=trial_number, + completed=completed, + accepted_patch=review.accepted_patch, + passed_checks=tuple( + name for name in manifest.task.checks if trial.checks[name].passed + ), + elapsed_seconds=trial.agent.elapsed_seconds, + cost_usd=trial.receipt.cost_usd, + human_interventions=review.human_interventions, + recovery_attempted=review.recovery_attempted, + recovery_succeeded=review.recovery_succeeded, + passed_handoff_assertions=tuple( + name + for name in manifest.task.handoff_assertions + if trial.handoff_assertions[name].passed + ), + ) + ) + + if round(total_cost, 6) > evidence.authorized_cost_cap_usd: + raise ValueError( + "normalized receipt total exceeds authorized cost cap " + f"(${total_cost:.6f} > ${evidence.authorized_cost_cap_usd:.6f})" + ) + return RecordedResults( + schema_version=SCHEMA_VERSION, + benchmark_id=manifest.benchmark_id, + task_id=manifest.task.id, + observations=tuple(observations), + ) + + +def finalized_results_json(results: RecordedResults) -> str: + return json.dumps(results.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) diff --git a/benchmarks/agent_cli_attestation.py b/benchmarks/agent_cli_attestation.py new file mode 100644 index 0000000..9a55138 --- /dev/null +++ b/benchmarks/agent_cli_attestation.py @@ -0,0 +1,452 @@ +"""Offline Ed25519 attestations for independent benchmark reviews.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from typing import Literal + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_adjudication import AdjudicationReviews, ReviewDecision +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_review_policy import ReviewerPolicy, validate_reviewer_separation + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode()).hexdigest() + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if not identifier or identifier != identifier.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (ValueError, TypeError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +class ReviewerPublicKeyDeclaration(_FrozenModel): + reviewer_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + status: Literal["active", "revoked"] = "active" + + @model_validator(mode="after") + def validate_key(self) -> ReviewerPublicKeyDeclaration: + _validate_identifier(self.reviewer_id, label="reviewer_id") + _validate_identifier(self.key_id, label="key_id") + _decode_base64(self.public_key_base64, label="public_key_base64", length=32) + return self + + +class ReviewerTrustDeclaration(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_authority_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + authority_root_ledger_sha256: str | None = Field( + default=None, + pattern=_SHA256_PATTERN, + ) + keys: tuple[ReviewerPublicKeyDeclaration, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_declaration(self) -> ReviewerTrustDeclaration: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + identities = [(key.reviewer_id, key.key_id) for key in self.keys] + if len(identities) != len(set(identities)): + raise ValueError("reviewer key identities must be unique") + if len({key.key_id for key in self.keys}) != len(self.keys): + raise ValueError("key_id values must be globally unique") + if ( + self.authority_root_ledger_sha256 is not None + and self.reviewer_authority_sha256 is None + ): + raise ValueError("authority root ledger requires a reviewer authority") + return self + + +class ReviewerPublicKey(_FrozenModel): + reviewer_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + public_key_sha256: str = Field(pattern=_SHA256_PATTERN) + status: Literal["active", "revoked"] + + @model_validator(mode="after") + def validate_fingerprint(self) -> ReviewerPublicKey: + declaration = ReviewerPublicKeyDeclaration( + reviewer_id=self.reviewer_id, + key_id=self.key_id, + algorithm=self.algorithm, + public_key_base64=self.public_key_base64, + status=self.status, + ) + decoded = _decode_base64( + declaration.public_key_base64, + label="public_key_base64", + length=32, + ) + if hashlib.sha256(decoded).hexdigest() != self.public_key_sha256: + raise ValueError("public key fingerprint does not match key") + return self + + +class ReviewerTrust(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_authority_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + authority_root_ledger_sha256: str | None = Field( + default=None, + pattern=_SHA256_PATTERN, + ) + keys: tuple[ReviewerPublicKey, ...] = Field(min_length=1) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_fingerprint(self) -> ReviewerTrust: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + ReviewerTrustDeclaration( + schema_version=self.schema_version, + benchmark_id=self.benchmark_id, + review_policy_sha256=self.review_policy_sha256, + reviewer_authority_sha256=self.reviewer_authority_sha256, + authority_root_ledger_sha256=self.authority_root_ledger_sha256, + keys=tuple( + ReviewerPublicKeyDeclaration( + reviewer_id=key.reviewer_id, + key_id=key.key_id, + algorithm=key.algorithm, + public_key_base64=key.public_key_base64, + status=key.status, + ) + for key in self.keys + ), + ) + if tuple(sorted(self.keys, key=lambda key: (key.reviewer_id, key.key_id))) != self.keys: + raise ValueError("reviewer keys must be sorted") + if self.reviewer_trust_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("reviewer trust fingerprint does not match trust declaration") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump( + mode="json", + exclude={"reviewer_trust_sha256"}, + exclude_none=True, + ) + + def to_json(self) -> str: + return json.dumps( + self.model_dump(mode="json", exclude_none=True), + ensure_ascii=False, + sort_keys=True, + ) + + +def build_reviewer_trust( + declaration: ReviewerTrustDeclaration, + policy: ReviewerPolicy, + *, + require_active_key_per_reviewer: bool = True, +) -> ReviewerTrust: + """Normalize declared Ed25519 trust roots and bind them to one review policy.""" + if declaration.benchmark_id != policy.benchmark_id: + raise ValueError("reviewer trust benchmark_id does not match review policy") + if declaration.review_policy_sha256 != policy.policy_sha256: + raise ValueError("reviewer trust policy fingerprint does not match review policy") + allowed = set(policy.reviewer_ids) + declared = {key.reviewer_id for key in declaration.keys} + unauthorized = sorted(declared - allowed) + if unauthorized: + raise ValueError( + "reviewer trust key is not for an allowed reviewer: " + f"{', '.join(unauthorized)}" + ) + if require_active_key_per_reviewer: + active = {key.reviewer_id for key in declaration.keys if key.status == "active"} + missing = sorted(allowed - active) + if missing: + raise ValueError(f"allowed reviewer has no active key: {', '.join(missing)}") + keys = tuple( + sorted( + ( + ReviewerPublicKey( + reviewer_id=key.reviewer_id, + key_id=key.key_id, + algorithm=key.algorithm, + public_key_base64=key.public_key_base64, + public_key_sha256=hashlib.sha256( + _decode_base64( + key.public_key_base64, + label="public_key_base64", + length=32, + ) + ).hexdigest(), + status=key.status, + ) + for key in declaration.keys + ), + key=lambda key: (key.reviewer_id, key.key_id), + ) + ) + payload = { + "schema_version": declaration.schema_version, + "benchmark_id": declaration.benchmark_id, + "review_policy_sha256": declaration.review_policy_sha256, + "keys": [key.model_dump(mode="json") for key in keys], + } + if declaration.reviewer_authority_sha256 is not None: + payload["reviewer_authority_sha256"] = declaration.reviewer_authority_sha256 + if declaration.authority_root_ledger_sha256 is not None: + if declaration.reviewer_authority_sha256 is None: + raise ValueError("authority root ledger requires a reviewer authority") + payload["authority_root_ledger_sha256"] = ( + declaration.authority_root_ledger_sha256 + ) + return ReviewerTrust(**payload, reviewer_trust_sha256=_canonical_sha256(payload)) + + +def completed_reviews_sha256(reviews: AdjudicationReviews) -> str: + """Fingerprint the exact completed review artifact.""" + if reviews.review_completed is not True: + raise ValueError("review attestations require review_completed=true") + return _canonical_sha256(reviews.model_dump(mode="json")) + + +def _reviewer_decisions_sha256( + decisions: tuple[ReviewDecision, ...], + reviewer_id: str, +) -> str: + owned = sorted( + ( + decision.model_dump(mode="json") + for decision in decisions + if decision.reviewer_id == reviewer_id + ), + key=lambda decision: (decision["arm"], decision["trial"]), + ) + if not owned: + raise ValueError(f"reviewer has no decisions: {reviewer_id}") + return _canonical_sha256(owned) + + +class ReviewerAttestationStatement(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + workspace_revision: str = Field(pattern=r"^[0-9a-f]{40}$") + preflight_sha256: str = Field(pattern=_SHA256_PATTERN) + evidence_sha256: str = Field(pattern=_SHA256_PATTERN) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + reviews_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_id: str = Field(min_length=1, max_length=200) + reviewer_decisions_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> ReviewerAttestationStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.reviewer_id, label="reviewer_id") + return self + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class AttestationSigningRequest(_FrozenModel): + statement: ReviewerAttestationStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_payload(self) -> AttestationSigningRequest: + expected = self.statement.signing_bytes() + try: + decoded = base64.b64decode(self.signing_payload_base64, validate=True) + except (ValueError, TypeError) as exc: + raise ValueError("signing payload must be canonical base64") from exc + if decoded != expected or base64.b64encode(decoded).decode() != self.signing_payload_base64: + raise ValueError("signing payload does not match attestation statement") + return self + + +class ReviewAttestationTemplate(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + reviews_sha256: str = Field(pattern=_SHA256_PATTERN) + requests: tuple[AttestationSigningRequest, ...] = Field(min_length=1) + attestations_completed: Literal[False] = False + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class SignedReviewAttestation(_FrozenModel): + statement: ReviewerAttestationStatement + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature_encoding(self) -> SignedReviewAttestation: + _validate_identifier(self.key_id, label="key_id") + _decode_base64(self.signature_base64, label="signature_base64", length=64) + return self + + +class ReviewAttestationBundle(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + reviews_sha256: str = Field(pattern=_SHA256_PATTERN) + attestations: tuple[SignedReviewAttestation, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_bundle(self) -> ReviewAttestationBundle: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + reviewer_ids = [item.statement.reviewer_id for item in self.attestations] + if len(reviewer_ids) != len(set(reviewer_ids)): + raise ValueError("review attestations must contain one signature per reviewer") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_review_attestation_template( + policy: ReviewerPolicy, + trust: ReviewerTrust, + reviews: AdjudicationReviews, +) -> ReviewAttestationTemplate: + """Create canonical signing payloads without reading or retaining private keys.""" + validate_reviewer_separation(policy, reviews) + if trust.benchmark_id != reviews.benchmark_id: + raise ValueError("reviewer trust benchmark_id does not match reviews") + if trust.review_policy_sha256 != policy.policy_sha256: + raise ValueError("reviewer trust policy fingerprint does not match review policy") + if reviews.reviewer_trust_sha256 != trust.reviewer_trust_sha256: + raise ValueError("reviewer trust fingerprint does not match reviews") + if reviews.preflight_sha256 is None or reviews.evidence_sha256 is None: + raise ValueError("attested reviews require preflight and evidence bindings") + reviews_sha256 = completed_reviews_sha256(reviews) + reviewer_ids = sorted({decision.reviewer_id for decision in reviews.decisions}) + requests = [] + for reviewer_id in reviewer_ids: + statement = ReviewerAttestationStatement( + schema_version=SCHEMA_VERSION, + benchmark_id=reviews.benchmark_id, + task_id=reviews.task_id, + workspace_revision=reviews.workspace_revision, + preflight_sha256=reviews.preflight_sha256, + evidence_sha256=reviews.evidence_sha256, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=trust.reviewer_trust_sha256, + reviews_sha256=reviews_sha256, + reviewer_id=reviewer_id, + reviewer_decisions_sha256=_reviewer_decisions_sha256( + reviews.decisions, + reviewer_id, + ), + ) + requests.append( + AttestationSigningRequest( + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + ) + return ReviewAttestationTemplate( + schema_version=SCHEMA_VERSION, + benchmark_id=reviews.benchmark_id, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=trust.reviewer_trust_sha256, + reviews_sha256=reviews_sha256, + requests=tuple(requests), + ) + + +def verify_review_attestations( + policy: ReviewerPolicy, + trust: ReviewerTrust, + reviews: AdjudicationReviews, + bundle: ReviewAttestationBundle, +) -> None: + """Verify complete reviewer coverage and every detached Ed25519 signature.""" + expected = build_review_attestation_template(policy, trust, reviews) + if bundle.benchmark_id != expected.benchmark_id: + raise ValueError("attestation benchmark_id does not match reviews") + if bundle.review_policy_sha256 != expected.review_policy_sha256: + raise ValueError("attestation policy fingerprint does not match reviews") + if bundle.reviewer_trust_sha256 != expected.reviewer_trust_sha256: + raise ValueError("attestation trust fingerprint does not match reviews") + if bundle.reviews_sha256 != expected.reviews_sha256: + raise ValueError("attestation reviews fingerprint does not match reviews") + signed_by_reviewer = { + attestation.statement.reviewer_id: attestation + for attestation in bundle.attestations + } + expected_by_reviewer = { + request.statement.reviewer_id: request.statement for request in expected.requests + } + if set(signed_by_reviewer) != set(expected_by_reviewer): + raise ValueError("attestation reviewer coverage does not match completed reviews") + keys = {(key.reviewer_id, key.key_id): key for key in trust.keys} + for reviewer_id, statement in expected_by_reviewer.items(): + attestation = signed_by_reviewer[reviewer_id] + if attestation.statement != statement: + raise ValueError(f"attestation statement does not match reviews: {reviewer_id}") + key = keys.get((reviewer_id, attestation.key_id)) + if key is None: + raise ValueError(f"attestation signing key is not trusted: {reviewer_id}") + if key.status != "active": + raise ValueError(f"attestation signing key is revoked: {reviewer_id}") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64( + key.public_key_base64, + label="public_key_base64", + length=32, + ) + ).verify( + _decode_base64( + attestation.signature_base64, + label="signature_base64", + length=64, + ), + statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError(f"attestation signature is invalid: {reviewer_id}") from exc diff --git a/benchmarks/agent_cli_authority.py b/benchmarks/agent_cli_authority.py new file mode 100644 index 0000000..d4eca17 --- /dev/null +++ b/benchmarks/agent_cli_authority.py @@ -0,0 +1,602 @@ +"""Offline organization authority for reviewer enrollment and campaign envelopes.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from typing import TYPE_CHECKING, Literal + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_adjudication import ( + AdjudicationReviews, + RecordedEvidence, + finalize_recorded_results, +) +from benchmarks.agent_cli_attestation import ( + ReviewAttestationBundle, + ReviewerPublicKey, + ReviewerTrust, + completed_reviews_sha256, +) +from benchmarks.agent_cli_comparison import ( + SCHEMA_VERSION, + AgentCliManifest, + RecordedResults, +) +from benchmarks.agent_cli_preflight import CampaignPreflight, validate_review_bindings +from benchmarks.agent_cli_review_policy import ReviewerPolicy + +if TYPE_CHECKING: + from benchmarks.agent_cli_transparency import SignedAuthorityRootLedger + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode()).hexdigest() + + +def _model_sha256(model: BaseModel) -> str: + return _canonical_sha256(model.model_dump(mode="json")) + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (ValueError, TypeError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if not identifier or identifier != identifier.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +class BenchmarkAuthorityDeclaration(_FrozenModel): + schema_version: int + authority_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_declaration(self) -> BenchmarkAuthorityDeclaration: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.authority_id, label="authority_id") + _decode_base64(self.public_key_base64, label="public_key_base64", length=32) + return self + + +class BenchmarkAuthority(_FrozenModel): + schema_version: int + authority_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + public_key_sha256: str = Field(pattern=_SHA256_PATTERN) + authority_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_fingerprints(self) -> BenchmarkAuthority: + BenchmarkAuthorityDeclaration( + schema_version=self.schema_version, + authority_id=self.authority_id, + algorithm=self.algorithm, + public_key_base64=self.public_key_base64, + ) + public_key = _decode_base64( + self.public_key_base64, + label="public_key_base64", + length=32, + ) + if self.public_key_sha256 != hashlib.sha256(public_key).hexdigest(): + raise ValueError("authority public key fingerprint does not match key") + if self.authority_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("authority fingerprint does not match declaration") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"authority_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_benchmark_authority( + declaration: BenchmarkAuthorityDeclaration, +) -> BenchmarkAuthority: + """Normalize one out-of-band organization authority public key.""" + public_key = _decode_base64( + declaration.public_key_base64, + label="public_key_base64", + length=32, + ) + payload = { + **declaration.model_dump(mode="json"), + "public_key_sha256": hashlib.sha256(public_key).hexdigest(), + } + return BenchmarkAuthority(**payload, authority_sha256=_canonical_sha256(payload)) + + +class ReviewerEnrollmentStatement(_FrozenModel): + schema_version: int + authority_sha256: str = Field(pattern=_SHA256_PATTERN) + benchmark_id: str = Field(min_length=1) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + reviewer_public_key_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> ReviewerEnrollmentStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.reviewer_id, label="reviewer_id") + _validate_identifier(self.key_id, label="key_id") + return self + + def signing_bytes(self) -> bytes: + return _canonical_json( + self.model_dump(mode="json", exclude_none=True) + ).encode() + + +class ReviewerEnrollmentSigningRequest(_FrozenModel): + statement: ReviewerEnrollmentStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_payload(self) -> ReviewerEnrollmentSigningRequest: + expected = self.statement.signing_bytes() + try: + decoded = base64.b64decode(self.signing_payload_base64, validate=True) + except (ValueError, TypeError) as exc: + raise ValueError("reviewer enrollment payload must be canonical base64") from exc + if decoded != expected or base64.b64encode(decoded).decode() != self.signing_payload_base64: + raise ValueError("reviewer enrollment payload does not match statement") + return self + + +class ReviewerEnrollmentTemplate(_FrozenModel): + schema_version: int + authority_sha256: str = Field(pattern=_SHA256_PATTERN) + benchmark_id: str = Field(min_length=1) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + requests: tuple[ReviewerEnrollmentSigningRequest, ...] = Field(min_length=1) + enrollments_completed: Literal[False] = False + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class ReviewerEnrollmentCertificate(_FrozenModel): + statement: ReviewerEnrollmentStatement + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature_encoding(self) -> ReviewerEnrollmentCertificate: + _decode_base64(self.signature_base64, label="signature_base64", length=64) + return self + + +class ReviewerEnrollmentBundle(_FrozenModel): + schema_version: int + authority_sha256: str = Field(pattern=_SHA256_PATTERN) + benchmark_id: str = Field(min_length=1) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + certificates: tuple[ReviewerEnrollmentCertificate, ...] = Field(min_length=1) + reviewer_enrollments_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_bundle(self) -> ReviewerEnrollmentBundle: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + identities = [ + (certificate.statement.reviewer_id, certificate.statement.key_id) + for certificate in self.certificates + ] + if len(identities) != len(set(identities)): + raise ValueError("reviewer enrollment certificates must be unique") + if tuple(sorted(self.certificates, key=_certificate_identity)) != self.certificates: + raise ValueError("reviewer enrollment certificates must be sorted") + if self.reviewer_enrollments_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("reviewer enrollment fingerprint does not match bundle") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"reviewer_enrollments_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _certificate_identity( + certificate: ReviewerEnrollmentCertificate, +) -> tuple[str, str]: + return certificate.statement.reviewer_id, certificate.statement.key_id + + +def build_reviewer_enrollment_statement( + authority: BenchmarkAuthority, + policy: ReviewerPolicy, + trust: ReviewerTrust, + key: ReviewerPublicKey, +) -> ReviewerEnrollmentStatement: + """Bind one reviewer public key to the authority, policy, and exact trust artifact.""" + if trust.reviewer_authority_sha256 != authority.authority_sha256: + raise ValueError("reviewer trust authority fingerprint does not match authority") + if trust.benchmark_id != policy.benchmark_id: + raise ValueError("reviewer trust benchmark_id does not match review policy") + if trust.review_policy_sha256 != policy.policy_sha256: + raise ValueError("reviewer trust policy fingerprint does not match review policy") + if key not in trust.keys: + raise ValueError("reviewer key is not present in trust") + return ReviewerEnrollmentStatement( + schema_version=SCHEMA_VERSION, + authority_sha256=authority.authority_sha256, + benchmark_id=trust.benchmark_id, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=trust.reviewer_trust_sha256, + reviewer_id=key.reviewer_id, + key_id=key.key_id, + algorithm=key.algorithm, + reviewer_public_key_sha256=key.public_key_sha256, + ) + + +def build_reviewer_enrollment_template( + authority: BenchmarkAuthority, + policy: ReviewerPolicy, + trust: ReviewerTrust, +) -> ReviewerEnrollmentTemplate: + """Create authority signing payloads for every key without reading a private key.""" + requests = [] + for key in trust.keys: + statement = build_reviewer_enrollment_statement( + authority, + policy, + trust, + key, + ) + requests.append( + ReviewerEnrollmentSigningRequest( + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + ) + return ReviewerEnrollmentTemplate( + schema_version=SCHEMA_VERSION, + authority_sha256=authority.authority_sha256, + benchmark_id=trust.benchmark_id, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=trust.reviewer_trust_sha256, + requests=tuple(requests), + ) + + +def build_reviewer_enrollment_bundle( + authority: BenchmarkAuthority, + policy: ReviewerPolicy, + trust: ReviewerTrust, + certificates: tuple[ReviewerEnrollmentCertificate, ...], +) -> ReviewerEnrollmentBundle: + """Normalize and verify organization-signed reviewer key enrollments.""" + ordered = tuple(sorted(certificates, key=_certificate_identity)) + payload = { + "schema_version": SCHEMA_VERSION, + "authority_sha256": authority.authority_sha256, + "benchmark_id": trust.benchmark_id, + "review_policy_sha256": policy.policy_sha256, + "reviewer_trust_sha256": trust.reviewer_trust_sha256, + "certificates": [certificate.model_dump(mode="json") for certificate in ordered], + } + bundle = ReviewerEnrollmentBundle( + **payload, + reviewer_enrollments_sha256=_canonical_sha256(payload), + ) + verify_reviewer_enrollments(authority, policy, trust, bundle) + return bundle + + +def verify_reviewer_enrollments( + authority: BenchmarkAuthority, + policy: ReviewerPolicy, + trust: ReviewerTrust, + bundle: ReviewerEnrollmentBundle, +) -> None: + """Require an authority signature for every key in the exact reviewer trust.""" + if bundle.reviewer_enrollments_sha256 != _canonical_sha256( + bundle._binding_payload() + ): + raise ValueError("reviewer enrollment fingerprint does not match bundle") + if trust.reviewer_authority_sha256 != authority.authority_sha256: + raise ValueError("reviewer trust authority fingerprint does not match authority") + expected_headers = ( + authority.authority_sha256, + trust.benchmark_id, + policy.policy_sha256, + trust.reviewer_trust_sha256, + ) + actual_headers = ( + bundle.authority_sha256, + bundle.benchmark_id, + bundle.review_policy_sha256, + bundle.reviewer_trust_sha256, + ) + if actual_headers != expected_headers: + raise ValueError("reviewer enrollment trust fingerprint does not match campaign") + certificates = {_certificate_identity(item): item for item in bundle.certificates} + keys = {(key.reviewer_id, key.key_id): key for key in trust.keys} + if set(certificates) != set(keys): + raise ValueError("reviewer enrollment certificate coverage does not match trust") + public_key = Ed25519PublicKey.from_public_bytes( + _decode_base64( + authority.public_key_base64, + label="public_key_base64", + length=32, + ) + ) + for identity, key in keys.items(): + certificate = certificates[identity] + expected_statement = build_reviewer_enrollment_statement( + authority, + policy, + trust, + key, + ) + if certificate.statement != expected_statement: + raise ValueError(f"reviewer enrollment statement does not match trust: {key.key_id}") + try: + public_key.verify( + _decode_base64( + certificate.signature_base64, + label="signature_base64", + length=64, + ), + certificate.statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError(f"reviewer enrollment signature is invalid: {key.key_id}") from exc + + +class CampaignEnvelopeStatement(_FrozenModel): + schema_version: int + authority_sha256: str = Field(pattern=_SHA256_PATTERN) + authority_root_ledger_sha256: str | None = Field( + default=None, + pattern=_SHA256_PATTERN, + ) + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + workspace_revision: str = Field(pattern=r"^[0-9a-f]{40}$") + manifest_sha256: str = Field(pattern=_SHA256_PATTERN) + preflight_sha256: str = Field(pattern=_SHA256_PATTERN) + evidence_sha256: str = Field(pattern=_SHA256_PATTERN) + reviews_sha256: str = Field(pattern=_SHA256_PATTERN) + review_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + reviewer_enrollments_sha256: str = Field(pattern=_SHA256_PATTERN) + attestations_sha256: str = Field(pattern=_SHA256_PATTERN) + results_sha256: str = Field(pattern=_SHA256_PATTERN) + paid_execution_authorized: Literal[False] = False + envelope_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> CampaignEnvelopeStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + expected = _canonical_sha256( + self.model_dump( + mode="json", + exclude={"envelope_sha256"}, + exclude_none=True, + ) + ) + if self.envelope_sha256 != expected: + raise ValueError("campaign envelope fingerprint does not match artifacts") + return self + + def signing_bytes(self) -> bytes: + return _canonical_json( + self.model_dump(mode="json", exclude_none=True) + ).encode() + + +class CampaignEnvelopeSigningRequest(_FrozenModel): + statement: CampaignEnvelopeStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_payload(self) -> CampaignEnvelopeSigningRequest: + expected = self.statement.signing_bytes() + try: + decoded = base64.b64decode(self.signing_payload_base64, validate=True) + except (ValueError, TypeError) as exc: + raise ValueError("campaign envelope payload must be canonical base64") from exc + if decoded != expected or base64.b64encode(decoded).decode() != self.signing_payload_base64: + raise ValueError("campaign envelope payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class SignedCampaignEnvelope(_FrozenModel): + statement: CampaignEnvelopeStatement + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature_encoding(self) -> SignedCampaignEnvelope: + _decode_base64(self.signature_base64, label="signature_base64", length=64) + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _validate_manifest_preflight( + manifest: AgentCliManifest, + preflight: CampaignPreflight, +) -> None: + identity = (manifest.benchmark_id, manifest.task.id, manifest.task.workspace_revision) + if (preflight.benchmark_id, preflight.task_id, preflight.workspace_revision) != identity: + raise ValueError("preflight identity does not match manifest") + if preflight.manifest_sha256 != _model_sha256(manifest): + raise ValueError("preflight manifest fingerprint does not match manifest") + + +def build_campaign_envelope_request( + *, + authority: BenchmarkAuthority, + manifest: AgentCliManifest, + preflight: CampaignPreflight, + evidence: RecordedEvidence, + reviews: AdjudicationReviews, + review_policy: ReviewerPolicy, + reviewer_trust: ReviewerTrust, + reviewer_enrollments: ReviewerEnrollmentBundle, + attestations: ReviewAttestationBundle, + results: RecordedResults, + authority_root_ledger: SignedAuthorityRootLedger | None = None, +) -> CampaignEnvelopeSigningRequest: + """Bind a validated campaign into one non-authorizing authority signing payload.""" + _validate_manifest_preflight(manifest, preflight) + validate_review_bindings(preflight, evidence, reviews) + verify_reviewer_enrollments( + authority, + review_policy, + reviewer_trust, + reviewer_enrollments, + ) + if reviewer_trust.authority_root_ledger_sha256 is not None: + if authority_root_ledger is None: + raise ValueError("authority root ledger is required for ledger-bound trust") + from benchmarks.agent_cli_transparency import verify_authority_root_ledger + + active_authority = verify_authority_root_ledger(authority_root_ledger) + if active_authority != authority: + raise ValueError("reviewer authority is not the active authority root") + if ( + authority_root_ledger.statement.ledger_sha256 + != reviewer_trust.authority_root_ledger_sha256 + ): + raise ValueError("reviewer trust authority root ledger does not match ledger") + elif authority_root_ledger is not None: + raise ValueError("authority root ledger requires ledger-bound trust") + expected_results = finalize_recorded_results( + manifest, + evidence, + reviews, + review_policy=review_policy, + reviewer_trust=reviewer_trust, + attestations=attestations, + reviewer_authority=authority, + reviewer_enrollments=reviewer_enrollments, + authority_root_ledger=authority_root_ledger, + ) + if results != expected_results: + raise ValueError("results do not match authority-anchored campaign artifacts") + payload = { + "schema_version": SCHEMA_VERSION, + "authority_sha256": authority.authority_sha256, + "benchmark_id": manifest.benchmark_id, + "task_id": manifest.task.id, + "workspace_revision": manifest.task.workspace_revision, + "manifest_sha256": _model_sha256(manifest), + "preflight_sha256": preflight.preflight_sha256, + "evidence_sha256": _model_sha256(evidence), + "reviews_sha256": completed_reviews_sha256(reviews), + "review_policy_sha256": review_policy.policy_sha256, + "reviewer_trust_sha256": reviewer_trust.reviewer_trust_sha256, + "reviewer_enrollments_sha256": ( + reviewer_enrollments.reviewer_enrollments_sha256 + ), + "attestations_sha256": _model_sha256(attestations), + "results_sha256": _model_sha256(results), + "paid_execution_authorized": False, + } + if authority_root_ledger is not None: + payload["authority_root_ledger_sha256"] = ( + authority_root_ledger.statement.ledger_sha256 + ) + statement = CampaignEnvelopeStatement( + **payload, + envelope_sha256=_canonical_sha256(payload), + ) + return CampaignEnvelopeSigningRequest( + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def verify_signed_campaign_envelope( + *, + authority: BenchmarkAuthority, + manifest: AgentCliManifest, + preflight: CampaignPreflight, + evidence: RecordedEvidence, + reviews: AdjudicationReviews, + review_policy: ReviewerPolicy, + reviewer_trust: ReviewerTrust, + reviewer_enrollments: ReviewerEnrollmentBundle, + attestations: ReviewAttestationBundle, + results: RecordedResults, + envelope: SignedCampaignEnvelope, + authority_root_ledger: SignedAuthorityRootLedger | None = None, +) -> None: + """Verify the authority signature over the complete finalized campaign chain.""" + expected = build_campaign_envelope_request( + authority=authority, + manifest=manifest, + preflight=preflight, + evidence=evidence, + reviews=reviews, + review_policy=review_policy, + reviewer_trust=reviewer_trust, + reviewer_enrollments=reviewer_enrollments, + attestations=attestations, + results=results, + authority_root_ledger=authority_root_ledger, + ) + if envelope.statement != expected.statement: + raise ValueError("signed campaign envelope does not match campaign artifacts") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64( + authority.public_key_base64, + label="public_key_base64", + length=32, + ) + ).verify( + _decode_base64( + envelope.signature_base64, + label="signature_base64", + length=64, + ), + envelope.statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError("campaign envelope signature is invalid") from exc diff --git a/benchmarks/agent_cli_campaign.py b/benchmarks/agent_cli_campaign.py new file mode 100644 index 0000000..9c444c3 --- /dev/null +++ b/benchmarks/agent_cli_campaign.py @@ -0,0 +1,501 @@ +"""Read-only lifecycle status for recorded agent CLI benchmark campaigns.""" + +from __future__ import annotations + +import hashlib +import json +from enum import Enum +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from benchmarks.agent_cli_adjudication import ( + AdjudicationReviews, + RecordedEvidence, + finalize_recorded_results, + recorded_evidence_sha256, +) +from benchmarks.agent_cli_attestation import ( + ReviewAttestationBundle, + ReviewerTrust, + verify_review_attestations, +) +from benchmarks.agent_cli_authority import ( + BenchmarkAuthority, + ReviewerEnrollmentBundle, + SignedCampaignEnvelope, + verify_reviewer_enrollments, + verify_signed_campaign_envelope, +) +from benchmarks.agent_cli_comparison import ( + SCHEMA_VERSION, + AgentCliManifest, + RecordedResults, +) +from benchmarks.agent_cli_preflight import ( + CampaignPreflight, + ReviewTemplate, + build_review_template, + validate_review_bindings, +) +from benchmarks.agent_cli_review_policy import ( + ReviewerPolicy, + validate_reviewer_policy_capacity, + validate_reviewer_separation, +) +from benchmarks.agent_cli_transparency import ( + SignedAuthorityRootLedger, + TransparencyConsistencyProof, + TransparencyInclusionProof, + verify_authority_root_ledger, + verify_transparency_consistency_proof, + verify_transparency_inclusion_proof, +) +from benchmarks.agent_cli_witness import ( + SignedWitnessCheckpoint, + TransparencyWitnessTrust, + verify_witness_checkpoint_bundle, +) + + +class CampaignStage(str, Enum): + MANIFEST_READY = "manifest_ready" + PREFLIGHT_READY = "preflight_ready" + RECORDED = "recorded" + REVIEW_PENDING = "review_pending" + AUTHORITY_ROOT_PENDING = "authority_root_pending" + REVIEWER_ENROLLMENT_PENDING = "reviewer_enrollment_pending" + REVIEW_ATTESTATION_PENDING = "review_attestation_pending" + REVIEW_COMPLETE = "review_complete" + CAMPAIGN_ENVELOPE_PENDING = "campaign_envelope_pending" + TRANSPARENCY_PENDING = "transparency_pending" + WITNESS_PENDING = "witness_pending" + FINALIZED = "finalized" + + +class CampaignStatus(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + schema_version: int + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + workspace_revision: str = Field(min_length=1) + stage: CampaignStage + has_preflight: bool + has_evidence: bool + has_review: bool + has_reviewer_enrollments: bool + has_attestations: bool + has_results: bool + has_campaign_envelope: bool + has_authority_root_ledger: bool + has_transparency_proof: bool + has_consistency_proof: bool + has_witness_trust: bool + has_witness_checkpoint: bool + preflight_sha256: str | None = None + evidence_sha256: str | None = None + review_policy_sha256: str | None = None + reviewer_trust_sha256: str | None = None + reviewer_authority_sha256: str | None = None + authority_root_ledger_sha256: str | None = None + reviewer_enrollments_sha256: str | None = None + transparency_consistency_proof_sha256: str | None = None + transparency_witness_trust_sha256: str | None = None + witness_checkpoint_sha256: str | None = None + attestations_verified: bool + campaign_envelope_verified: bool + authority_root_ledger_verified: bool + transparency_inclusion_verified: bool + transparency_consistency_verified: bool + witness_checkpoint_verified: bool + paid_execution_authorized: Literal[False] = False + next_action: str = Field(min_length=1) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _validate_preflight_manifest( + manifest: AgentCliManifest, + preflight: CampaignPreflight, +) -> None: + identity = (manifest.benchmark_id, manifest.task.id, manifest.task.workspace_revision) + if ( + preflight.benchmark_id, + preflight.task_id, + preflight.workspace_revision, + ) != identity: + raise ValueError("preflight identity does not match manifest") + manifest_body = json.dumps( + manifest.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + if preflight.manifest_sha256 != hashlib.sha256(manifest_body.encode()).hexdigest(): + raise ValueError("preflight manifest fingerprint does not match manifest") + + +def _validate_template( + preflight: CampaignPreflight, + evidence: RecordedEvidence, + template: ReviewTemplate, + policy: ReviewerPolicy | None, + trust: ReviewerTrust | None, +) -> None: + if policy is not None: + validate_reviewer_policy_capacity(policy, decision_count=len(template.decisions)) + if trust is not None and policy is None: + raise ValueError("review policy is required with reviewer trust") + if trust is not None and policy is not None: + if trust.benchmark_id != policy.benchmark_id: + raise ValueError("reviewer trust benchmark_id does not match review policy") + if trust.review_policy_sha256 != policy.policy_sha256: + raise ValueError("reviewer trust policy fingerprint does not match review policy") + expected = build_review_template( + preflight, + evidence, + review_policy_sha256=(policy.policy_sha256 if policy is not None else None), + reviewer_trust_sha256=( + trust.reviewer_trust_sha256 if trust is not None else None + ), + ) + if template != expected: + raise ValueError("review template does not match campaign artifacts") + + +def build_campaign_status( + manifest: AgentCliManifest, + *, + preflight: CampaignPreflight | None = None, + evidence: RecordedEvidence | None = None, + review_template: ReviewTemplate | None = None, + reviews: AdjudicationReviews | None = None, + results: RecordedResults | None = None, + review_policy: ReviewerPolicy | None = None, + reviewer_trust: ReviewerTrust | None = None, + attestations: ReviewAttestationBundle | None = None, + reviewer_authority: BenchmarkAuthority | None = None, + reviewer_enrollments: ReviewerEnrollmentBundle | None = None, + campaign_envelope: SignedCampaignEnvelope | None = None, + authority_root_ledger: SignedAuthorityRootLedger | None = None, + transparency_proof: TransparencyInclusionProof | None = None, + transparency_consistency_proof: TransparencyConsistencyProof | None = None, + transparency_witness_trust: TransparencyWitnessTrust | None = None, + witness_checkpoint: SignedWitnessCheckpoint | None = None, +) -> CampaignStatus: + """Validate supplied artifacts and report the furthest complete lifecycle stage.""" + if evidence is not None and preflight is None: + raise ValueError("preflight is required before evidence") + if (review_template is not None or reviews is not None) and evidence is None: + raise ValueError("evidence is required before review") + if review_template is not None and reviews is not None: + raise ValueError("provide pending review template or completed reviews, not both") + if results is not None and reviews is None: + raise ValueError("completed reviews are required before results") + if attestations is not None and reviews is None: + raise ValueError("completed reviews are required before attestations") + if reviewer_enrollments is not None and reviewer_authority is None: + raise ValueError("reviewer authority is required before reviewer enrollments") + if campaign_envelope is not None and results is None: + raise ValueError("results are required before a signed campaign envelope") + if authority_root_ledger is not None and reviewer_authority is None: + raise ValueError("reviewer authority is required with an authority root ledger") + if transparency_proof is not None and campaign_envelope is None: + raise ValueError("signed campaign envelope is required before transparency proof") + if transparency_proof is not None and authority_root_ledger is None: + raise ValueError("authority root ledger is required before transparency proof") + if witness_checkpoint is not None and transparency_witness_trust is None: + raise ValueError("witness trust is required before witness checkpoint") + if witness_checkpoint is not None and transparency_consistency_proof is None: + raise ValueError("consistency proof is required before witness checkpoint") + if ( + transparency_consistency_proof is not None + and transparency_witness_trust is None + ): + raise ValueError("witness trust is required with a consistency proof") + if transparency_witness_trust is not None and ( + reviewer_trust is None + or reviewer_trust.authority_root_ledger_sha256 is None + ): + raise ValueError("witness trust requires ledger-bound reviewer trust") + + stage = CampaignStage.MANIFEST_READY + next_action = "create_preflight" + attestations_verified = False + campaign_envelope_verified = False + authority_root_ledger_verified = False + transparency_inclusion_verified = False + transparency_consistency_verified = False + witness_checkpoint_verified = False + if preflight is not None: + _validate_preflight_manifest(manifest, preflight) + stage = CampaignStage.PREFLIGHT_READY + next_action = "record_trials_with_explicit_consent" + if evidence is not None: + assert preflight is not None + if evidence.estimated_max_cost_usd != preflight.estimated_max_cost_usd: + raise ValueError("evidence estimate does not match preflight") + build_review_template(preflight, evidence) + stage = CampaignStage.RECORDED + next_action = "create_review_template" + if review_template is not None: + assert preflight is not None and evidence is not None + _validate_template( + preflight, + evidence, + review_template, + review_policy, + reviewer_trust, + ) + stage = CampaignStage.REVIEW_PENDING + next_action = "complete_independent_review" + if reviews is not None: + assert preflight is not None and evidence is not None + validate_review_bindings(preflight, evidence, reviews) + if review_policy is not None: + validate_reviewer_separation(review_policy, reviews) + elif reviews.review_policy_sha256 is not None: + raise ValueError("review policy is required for policy-bound reviews") + if reviews.reviewer_trust_sha256 is not None: + if reviewer_trust is None: + raise ValueError("reviewer trust is required for trust-bound reviews") + authority_bound = reviewer_trust.reviewer_authority_sha256 is not None + enrollments_verified = not authority_bound + if authority_bound: + if ( + reviewer_authority is not None + and reviewer_trust.reviewer_authority_sha256 + != reviewer_authority.authority_sha256 + ): + raise ValueError( + "reviewer trust authority fingerprint does not match authority" + ) + ledger_bound = reviewer_trust.authority_root_ledger_sha256 is not None + root_ready = True + if ledger_bound and authority_root_ledger is None: + stage = CampaignStage.AUTHORITY_ROOT_PENDING + next_action = "provide_authority_root_ledger" + root_ready = False + elif ledger_bound: + assert authority_root_ledger is not None + active = verify_authority_root_ledger(authority_root_ledger) + if active != reviewer_authority: + raise ValueError("reviewer authority is not the active authority root") + if ( + authority_root_ledger.statement.ledger_sha256 + != reviewer_trust.authority_root_ledger_sha256 + ): + raise ValueError( + "reviewer trust authority root ledger does not match ledger" + ) + authority_root_ledger_verified = True + elif authority_root_ledger is not None: + raise ValueError("authority root ledger requires ledger-bound trust") + if root_ready and ( + reviewer_authority is None or reviewer_enrollments is None + ): + stage = CampaignStage.REVIEWER_ENROLLMENT_PENDING + next_action = "collect_authority_enrollments" + elif root_ready: + assert review_policy is not None + verify_reviewer_enrollments( + reviewer_authority, + review_policy, + reviewer_trust, + reviewer_enrollments, + ) + enrollments_verified = True + elif reviewer_authority is not None or reviewer_enrollments is not None: + raise ValueError("reviewer enrollments require an authority-bound trust") + if enrollments_verified and attestations is None: + stage = CampaignStage.REVIEW_ATTESTATION_PENDING + next_action = "collect_reviewer_attestations" + elif enrollments_verified: + assert review_policy is not None + verify_review_attestations( + review_policy, + reviewer_trust, + reviews, + attestations, + ) + attestations_verified = True + stage = CampaignStage.REVIEW_COMPLETE + next_action = "finalize_results" + else: + if reviewer_trust is not None or attestations is not None: + raise ValueError("attestations require a reviewer trust binding") + stage = CampaignStage.REVIEW_COMPLETE + next_action = "finalize_results" + if results is not None: + assert evidence is not None and reviews is not None + expected_results = finalize_recorded_results( + manifest, + evidence, + reviews, + review_policy=review_policy, + reviewer_trust=reviewer_trust, + attestations=attestations, + reviewer_authority=reviewer_authority, + reviewer_enrollments=reviewer_enrollments, + authority_root_ledger=authority_root_ledger, + ) + if results != expected_results: + raise ValueError("results do not match finalized campaign artifacts") + if ( + reviewer_trust is not None + and reviewer_trust.reviewer_authority_sha256 is not None + ): + assert preflight is not None + assert review_policy is not None + assert reviewer_authority is not None + assert reviewer_enrollments is not None + assert attestations is not None + if campaign_envelope is None: + stage = CampaignStage.CAMPAIGN_ENVELOPE_PENDING + next_action = "sign_campaign_envelope" + else: + verify_signed_campaign_envelope( + authority=reviewer_authority, + manifest=manifest, + preflight=preflight, + evidence=evidence, + reviews=reviews, + review_policy=review_policy, + reviewer_trust=reviewer_trust, + reviewer_enrollments=reviewer_enrollments, + attestations=attestations, + results=results, + envelope=campaign_envelope, + authority_root_ledger=authority_root_ledger, + ) + campaign_envelope_verified = True + if reviewer_trust.authority_root_ledger_sha256 is not None: + assert authority_root_ledger is not None + if transparency_proof is None: + stage = CampaignStage.TRANSPARENCY_PENDING + next_action = "publish_campaign_envelope_to_transparency_log" + else: + verify_transparency_inclusion_proof( + transparency_proof, + authority_root_ledger, + expected_kind="campaign_envelope", + expected_artifact_sha256=( + campaign_envelope.statement.envelope_sha256 + ), + ) + transparency_inclusion_verified = True + if transparency_witness_trust is None: + stage = CampaignStage.FINALIZED + next_action = "campaign_complete" + elif ( + transparency_consistency_proof is None + or witness_checkpoint is None + ): + stage = CampaignStage.WITNESS_PENDING + next_action = "collect_witness_checkpoint" + else: + if ( + transparency_consistency_proof.current_tree_head + != transparency_proof.tree_head + ): + raise ValueError( + "consistency proof current tree head does not " + "match inclusion proof" + ) + verify_transparency_consistency_proof( + transparency_consistency_proof, + authority_root_ledger, + ) + transparency_consistency_verified = True + verify_witness_checkpoint_bundle( + transparency_witness_trust, + transparency_consistency_proof, + authority_root_ledger, + witness_checkpoint, + ) + witness_checkpoint_verified = True + stage = CampaignStage.FINALIZED + next_action = "campaign_complete" + else: + if transparency_proof is not None: + raise ValueError( + "transparency proof requires ledger-bound trust" + ) + stage = CampaignStage.FINALIZED + next_action = "campaign_complete" + else: + if campaign_envelope is not None: + raise ValueError("campaign envelope requires an authority-bound trust") + stage = CampaignStage.FINALIZED + next_action = "campaign_complete" + + return CampaignStatus( + schema_version=SCHEMA_VERSION, + benchmark_id=manifest.benchmark_id, + task_id=manifest.task.id, + workspace_revision=manifest.task.workspace_revision, + stage=stage, + has_preflight=preflight is not None, + has_evidence=evidence is not None, + has_review=review_template is not None or reviews is not None, + has_reviewer_enrollments=reviewer_enrollments is not None, + has_attestations=attestations is not None, + has_results=results is not None, + has_campaign_envelope=campaign_envelope is not None, + has_authority_root_ledger=authority_root_ledger is not None, + has_transparency_proof=transparency_proof is not None, + has_consistency_proof=transparency_consistency_proof is not None, + has_witness_trust=transparency_witness_trust is not None, + has_witness_checkpoint=witness_checkpoint is not None, + preflight_sha256=(preflight.preflight_sha256 if preflight is not None else None), + evidence_sha256=( + recorded_evidence_sha256(evidence) if evidence is not None else None + ), + review_policy_sha256=( + review_policy.policy_sha256 if review_policy is not None else None + ), + reviewer_trust_sha256=( + reviewer_trust.reviewer_trust_sha256 + if reviewer_trust is not None + else None + ), + reviewer_authority_sha256=( + reviewer_authority.authority_sha256 + if reviewer_authority is not None + else None + ), + authority_root_ledger_sha256=( + authority_root_ledger.statement.ledger_sha256 + if authority_root_ledger is not None + else None + ), + reviewer_enrollments_sha256=( + reviewer_enrollments.reviewer_enrollments_sha256 + if reviewer_enrollments is not None + else None + ), + transparency_consistency_proof_sha256=( + transparency_consistency_proof.consistency_proof_sha256 + if transparency_consistency_proof is not None + else None + ), + transparency_witness_trust_sha256=( + transparency_witness_trust.witness_trust_sha256 + if transparency_witness_trust is not None + else None + ), + witness_checkpoint_sha256=( + witness_checkpoint.witness_checkpoint_sha256 + if witness_checkpoint is not None + else None + ), + attestations_verified=attestations_verified, + campaign_envelope_verified=campaign_envelope_verified, + authority_root_ledger_verified=authority_root_ledger_verified, + transparency_inclusion_verified=transparency_inclusion_verified, + transparency_consistency_verified=transparency_consistency_verified, + witness_checkpoint_verified=witness_checkpoint_verified, + next_action=next_action, + ) diff --git a/benchmarks/agent_cli_checkpoint_registry.py b/benchmarks/agent_cli_checkpoint_registry.py new file mode 100644 index 0000000..569524a --- /dev/null +++ b/benchmarks/agent_cli_checkpoint_registry.py @@ -0,0 +1,1576 @@ +"""Durable witnessed-checkpoint registry and authenticated peer exchange.""" + +from __future__ import annotations + +import base64 +import fcntl +import hashlib +import json +import os +import stat +from pathlib import Path +from typing import Literal, Protocol + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_transparency import ( + SignedAuthorityRootLedger, + TransparencyConsistencyProof, +) +from benchmarks.agent_cli_witness import ( + SignedWitnessCheckpoint, + TransparencyWitnessTrust, + detect_witness_checkpoint_conflict, + verify_witness_checkpoint_bundle, +) + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode()).hexdigest() + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if not identifier or identifier != identifier.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +class CheckpointRegistryRecord(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + sequence: int = Field(ge=0) + previous_record_sha256: str | None = Field( + default=None, + pattern=_SHA256_PATTERN, + ) + authority_root_ledger_sha256: str = Field(pattern=_SHA256_PATTERN) + witness_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + consistency_proof: TransparencyConsistencyProof + checkpoint: SignedWitnessCheckpoint + record_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_record(self) -> CheckpointRegistryRecord: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + if self.record_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("checkpoint registry record fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"record_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointRegistrySnapshot(_FrozenModel): + schema_version: int + registry_id: str + records: tuple[CheckpointRegistryRecord, ...] + record_count: int = Field(ge=0) + head_record_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + current_tree_size: int | None = Field(default=None, ge=1) + current_root_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_snapshot(self) -> CheckpointRegistrySnapshot: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + if self.record_count != len(self.records): + raise ValueError("checkpoint snapshot record_count does not match records") + for sequence, record in enumerate(self.records): + if record.registry_id != self.registry_id or record.sequence != sequence: + raise ValueError("checkpoint snapshot record sequence is invalid") + expected_previous = ( + self.records[sequence - 1].record_sha256 if sequence else None + ) + if record.previous_record_sha256 != expected_previous: + raise ValueError("checkpoint snapshot hash chain is invalid") + last = self.records[-1] if self.records else None + expected = ( + last.record_sha256 if last else None, + last.checkpoint.statement.current_tree_size if last else None, + last.checkpoint.statement.current_root_sha256 if last else None, + ) + actual = ( + self.head_record_sha256, + self.current_tree_size, + self.current_root_sha256, + ) + if actual != expected: + raise ValueError("checkpoint snapshot head metadata does not match records") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointPeerKeyDeclaration(_FrozenModel): + peer_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + status: Literal["active", "revoked"] = "active" + + @model_validator(mode="after") + def validate_key(self) -> CheckpointPeerKeyDeclaration: + _validate_identifier(self.peer_id, label="peer_id") + _validate_identifier(self.key_id, label="key_id") + _decode_base64(self.public_key_base64, label="peer public key", length=32) + return self + + +class CheckpointPeerTrustDeclaration(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + keys: tuple[CheckpointPeerKeyDeclaration, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_declaration(self) -> CheckpointPeerTrustDeclaration: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + identities = [(key.peer_id, key.key_id) for key in self.keys] + if len(identities) != len(set(identities)): + raise ValueError("peer key identities must be unique") + if len({key.key_id for key in self.keys}) != len(self.keys): + raise ValueError("peer key_id values must be globally unique") + return self + + +class CheckpointPeerKey(_FrozenModel): + peer_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + public_key_sha256: str = Field(pattern=_SHA256_PATTERN) + status: Literal["active", "revoked"] + + @model_validator(mode="after") + def validate_key(self) -> CheckpointPeerKey: + declaration = CheckpointPeerKeyDeclaration( + peer_id=self.peer_id, + key_id=self.key_id, + algorithm=self.algorithm, + public_key_base64=self.public_key_base64, + status=self.status, + ) + decoded = _decode_base64( + declaration.public_key_base64, + label="peer public key", + length=32, + ) + if self.public_key_sha256 != hashlib.sha256(decoded).hexdigest(): + raise ValueError("peer public key fingerprint does not match key") + return self + + +def _peer_key_identity( + key: CheckpointPeerKey | CheckpointPeerKeyDeclaration, +) -> tuple[str, str]: + return key.peer_id, key.key_id + + +class CheckpointPeerTrust(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + keys: tuple[CheckpointPeerKey, ...] = Field(min_length=1) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_trust(self) -> CheckpointPeerTrust: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + if tuple(sorted(self.keys, key=_peer_key_identity)) != self.keys: + raise ValueError("peer keys must be sorted") + if len({key.key_id for key in self.keys}) != len(self.keys): + raise ValueError("peer key_id values must be globally unique") + declared_peers = {key.peer_id for key in self.keys} + active_peers = {key.peer_id for key in self.keys if key.status == "active"} + missing = sorted(declared_peers - active_peers) + if missing: + raise ValueError(f"peer has no active key: {', '.join(missing)}") + if self.peer_trust_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("peer trust fingerprint does not match declaration") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"peer_trust_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointPeerTrustResolver(Protocol): + def resolve_peer_trust(self, peer_trust_sha256: str) -> CheckpointPeerTrust: ... + + +type CheckpointPeerTrustSource = CheckpointPeerTrust | CheckpointPeerTrustResolver + + +def _resolve_peer_trust( + source: CheckpointPeerTrustSource, + peer_trust_sha256: str, +) -> CheckpointPeerTrust: + if isinstance(source, CheckpointPeerTrust): + if source.peer_trust_sha256 != peer_trust_sha256: + raise ValueError("checkpoint artifact does not match peer trust") + return source + resolved = source.resolve_peer_trust(peer_trust_sha256) + return CheckpointPeerTrust.model_validate(resolved.model_dump(mode="json")) + + +def build_checkpoint_peer_trust( + declaration: CheckpointPeerTrustDeclaration, +) -> CheckpointPeerTrust: + """Normalize an authenticated registry-peer key set.""" + keys = tuple( + sorted( + ( + CheckpointPeerKey( + peer_id=key.peer_id, + key_id=key.key_id, + algorithm=key.algorithm, + public_key_base64=key.public_key_base64, + public_key_sha256=hashlib.sha256( + _decode_base64( + key.public_key_base64, + label="peer public key", + length=32, + ) + ).hexdigest(), + status=key.status, + ) + for key in declaration.keys + ), + key=_peer_key_identity, + ) + ) + payload = { + "schema_version": declaration.schema_version, + "registry_id": declaration.registry_id, + "keys": [key.model_dump(mode="json") for key in keys], + } + return CheckpointPeerTrust.model_validate( + {**payload, "peer_trust_sha256": _canonical_sha256(payload)} + ) + + +class CheckpointExchangeStatement(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + source_peer_id: str = Field(min_length=1, max_length=200) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + record_sequence: int = Field(ge=0) + record_sha256: str = Field(pattern=_SHA256_PATTERN) + witness_checkpoint_sha256: str = Field(pattern=_SHA256_PATTERN) + log_id: str = Field(min_length=1, max_length=200) + current_tree_size: int = Field(ge=2) + current_root_sha256: str = Field(pattern=_SHA256_PATTERN) + exchange_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> CheckpointExchangeStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.source_peer_id, label="source_peer_id") + if self.exchange_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("checkpoint exchange fingerprint does not match statement") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"exchange_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class CheckpointExchangeSigningRequest(_FrozenModel): + source_peer_id: str + eligible_key_ids: tuple[str, ...] = Field(min_length=1) + statement: CheckpointExchangeStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_request(self) -> CheckpointExchangeSigningRequest: + if tuple(sorted(set(self.eligible_key_ids))) != self.eligible_key_ids: + raise ValueError("eligible peer key IDs must be sorted and unique") + expected = self.statement.signing_bytes() + decoded = _decode_base64( + self.signing_payload_base64, + label="checkpoint exchange signing payload", + length=len(expected), + ) + if decoded != expected: + raise ValueError("checkpoint exchange payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class SignedCheckpointExchangePacket(_FrozenModel): + schema_version: int + statement: CheckpointExchangeStatement + record: CheckpointRegistryRecord + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + packet_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_packet(self) -> SignedCheckpointExchangePacket: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.key_id, label="key_id") + _decode_base64( + self.signature_base64, + label="checkpoint exchange signature", + length=64, + ) + if self.packet_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("checkpoint exchange packet fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"packet_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _exchange_statement( + record: CheckpointRegistryRecord, + trust: CheckpointPeerTrust, + *, + source_peer_id: str, +) -> CheckpointExchangeStatement: + record = CheckpointRegistryRecord.model_validate(record.model_dump(mode="json")) + trust = CheckpointPeerTrust.model_validate(trust.model_dump(mode="json")) + if record.registry_id != trust.registry_id: + raise ValueError("checkpoint record does not match peer trust registry") + active = [key for key in trust.keys if key.peer_id == source_peer_id and key.status == "active"] + if not active: + raise ValueError("source peer has no active trusted key") + current = record.checkpoint.statement + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": record.registry_id, + "source_peer_id": source_peer_id, + "peer_trust_sha256": trust.peer_trust_sha256, + "record_sequence": record.sequence, + "record_sha256": record.record_sha256, + "witness_checkpoint_sha256": record.checkpoint.witness_checkpoint_sha256, + "log_id": current.log_id, + "current_tree_size": current.current_tree_size, + "current_root_sha256": current.current_root_sha256, + } + return CheckpointExchangeStatement.model_validate( + {**payload, "exchange_sha256": _canonical_sha256(payload)} + ) + + +def build_checkpoint_exchange_request( + record: CheckpointRegistryRecord, + trust: CheckpointPeerTrust, + *, + source_peer_id: str, +) -> CheckpointExchangeSigningRequest: + """Create a private-key-free request for one exact registry record.""" + statement = _exchange_statement(record, trust, source_peer_id=source_peer_id) + eligible = tuple( + sorted( + key.key_id + for key in trust.keys + if key.peer_id == source_peer_id and key.status == "active" + ) + ) + return CheckpointExchangeSigningRequest( + source_peer_id=source_peer_id, + eligible_key_ids=eligible, + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def build_signed_checkpoint_exchange_packet( + request: CheckpointExchangeSigningRequest, + record: CheckpointRegistryRecord, + *, + key_id: str, + signature_base64: str, + peer_trust: CheckpointPeerTrust, +) -> SignedCheckpointExchangePacket: + """Bind a detached peer signature to the exact exported record.""" + expected = build_checkpoint_exchange_request( + record, + peer_trust, + source_peer_id=request.source_peer_id, + ) + if request != expected: + raise ValueError("checkpoint exchange request does not match record") + if key_id not in request.eligible_key_ids: + raise ValueError("checkpoint exchange signature does not use an active trusted key") + payload = { + "schema_version": SCHEMA_VERSION, + "statement": request.statement.model_dump(mode="json"), + "record": record.model_dump(mode="json"), + "key_id": key_id, + "signature_base64": signature_base64, + } + packet = SignedCheckpointExchangePacket.model_validate( + {**payload, "packet_sha256": _canonical_sha256(payload)} + ) + verify_checkpoint_exchange_packet(packet, peer_trust) + return packet + + +def verify_checkpoint_exchange_packet( + packet: SignedCheckpointExchangePacket, + trust: CheckpointPeerTrust, +) -> CheckpointRegistryRecord: + """Authenticate a peer packet and its exact self-fingerprinted record.""" + packet = SignedCheckpointExchangePacket.model_validate(packet.model_dump(mode="json")) + trust = CheckpointPeerTrust.model_validate(trust.model_dump(mode="json")) + key = next( + ( + candidate + for candidate in trust.keys + if candidate.peer_id == packet.statement.source_peer_id + and candidate.key_id == packet.key_id + and candidate.status == "active" + ), + None, + ) + if key is None: + raise ValueError("checkpoint exchange signature does not use an active trusted key") + if packet.statement.peer_trust_sha256 != trust.peer_trust_sha256: + raise ValueError("checkpoint exchange does not match peer trust") + expected = _exchange_statement( + packet.record, + trust, + source_peer_id=packet.statement.source_peer_id, + ) + if packet.statement != expected: + raise ValueError("checkpoint exchange statement does not match record") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64(key.public_key_base64, label="peer public key", length=32) + ).verify( + _decode_base64( + packet.signature_base64, + label="checkpoint exchange signature", + length=64, + ), + packet.statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError("checkpoint exchange signature is invalid") from exc + return packet.record + + +class CheckpointRangeStatement(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + source_peer_id: str = Field(min_length=1, max_length=200) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + first_sequence: int = Field(ge=0) + last_sequence: int = Field(ge=0) + base_previous_record_sha256: str | None = Field( + default=None, + pattern=_SHA256_PATTERN, + ) + first_record_sha256: str = Field(pattern=_SHA256_PATTERN) + last_record_sha256: str = Field(pattern=_SHA256_PATTERN) + records_sha256: str = Field(pattern=_SHA256_PATTERN) + range_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> CheckpointRangeStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.source_peer_id, label="source_peer_id") + if self.first_sequence > self.last_sequence: + raise ValueError("checkpoint range sequence is inverted") + if self.range_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("checkpoint range fingerprint does not match statement") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"range_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class CheckpointRangeSigningRequest(_FrozenModel): + source_peer_id: str + eligible_key_ids: tuple[str, ...] = Field(min_length=1) + statement: CheckpointRangeStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_request(self) -> CheckpointRangeSigningRequest: + if self.source_peer_id != self.statement.source_peer_id: + raise ValueError("checkpoint range request source peer does not match statement") + if tuple(sorted(set(self.eligible_key_ids))) != self.eligible_key_ids: + raise ValueError("eligible range key IDs must be sorted and unique") + expected = self.statement.signing_bytes() + decoded = _decode_base64( + self.signing_payload_base64, + label="checkpoint range signing payload", + length=len(expected), + ) + if decoded != expected: + raise ValueError("checkpoint range payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class SignedCheckpointRangeBundle(_FrozenModel): + schema_version: int + statement: CheckpointRangeStatement + records: tuple[CheckpointRegistryRecord, ...] = Field(min_length=1) + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + range_bundle_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_bundle(self) -> SignedCheckpointRangeBundle: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.key_id, label="key_id") + _decode_base64( + self.signature_base64, + label="checkpoint range signature", + length=64, + ) + if self.range_bundle_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("checkpoint range bundle fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"range_bundle_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _validate_range_records( + records: tuple[CheckpointRegistryRecord, ...], +) -> tuple[CheckpointRegistryRecord, ...]: + if not records: + raise ValueError("checkpoint range requires at least one record") + normalized = tuple( + CheckpointRegistryRecord.model_validate(record.model_dump(mode="json")) + for record in records + ) + registry_id = normalized[0].registry_id + for index, record in enumerate(normalized): + if record.registry_id != registry_id: + raise ValueError("checkpoint range records use different registries") + if record.sequence != normalized[0].sequence + index: + raise ValueError("checkpoint range sequence is not contiguous") + if index and record.previous_record_sha256 != normalized[index - 1].record_sha256: + raise ValueError("checkpoint range hash chain is invalid") + return normalized + + +def _checkpoint_range_statement( + records: tuple[CheckpointRegistryRecord, ...], + trust: CheckpointPeerTrust, + *, + source_peer_id: str, +) -> CheckpointRangeStatement: + records = _validate_range_records(records) + trust = CheckpointPeerTrust.model_validate(trust.model_dump(mode="json")) + if records[0].registry_id != trust.registry_id: + raise ValueError("checkpoint range does not match peer trust registry") + active = tuple( + key + for key in trust.keys + if key.peer_id == source_peer_id and key.status == "active" + ) + if not active: + raise ValueError("source peer has no active trusted key") + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": records[0].registry_id, + "source_peer_id": source_peer_id, + "peer_trust_sha256": trust.peer_trust_sha256, + "first_sequence": records[0].sequence, + "last_sequence": records[-1].sequence, + "base_previous_record_sha256": records[0].previous_record_sha256, + "first_record_sha256": records[0].record_sha256, + "last_record_sha256": records[-1].record_sha256, + "records_sha256": _canonical_sha256( + [record.record_sha256 for record in records] + ), + } + return CheckpointRangeStatement.model_validate( + {**payload, "range_sha256": _canonical_sha256(payload)} + ) + + +def build_checkpoint_range_request( + records: tuple[CheckpointRegistryRecord, ...], + trust: CheckpointPeerTrust, + *, + source_peer_id: str, +) -> CheckpointRangeSigningRequest: + """Create one detached signing request for a contiguous record range.""" + statement = _checkpoint_range_statement( + records, + trust, + source_peer_id=source_peer_id, + ) + eligible = tuple( + sorted( + key.key_id + for key in trust.keys + if key.peer_id == source_peer_id and key.status == "active" + ) + ) + return CheckpointRangeSigningRequest( + source_peer_id=source_peer_id, + eligible_key_ids=eligible, + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def build_signed_checkpoint_range_bundle( + request: CheckpointRangeSigningRequest, + records: tuple[CheckpointRegistryRecord, ...], + *, + key_id: str, + signature_base64: str, + peer_trust: CheckpointPeerTrust, +) -> SignedCheckpointRangeBundle: + """Bind one peer signature to an exact contiguous record range.""" + expected = build_checkpoint_range_request( + records, + peer_trust, + source_peer_id=request.source_peer_id, + ) + if request != expected: + raise ValueError("checkpoint range request does not match records") + if key_id not in request.eligible_key_ids: + raise ValueError("checkpoint range signature does not use an active trusted key") + payload = { + "schema_version": SCHEMA_VERSION, + "statement": request.statement.model_dump(mode="json"), + "records": [record.model_dump(mode="json") for record in records], + "key_id": key_id, + "signature_base64": signature_base64, + } + bundle = SignedCheckpointRangeBundle.model_validate( + {**payload, "range_bundle_sha256": _canonical_sha256(payload)} + ) + verify_checkpoint_range_bundle(bundle, peer_trust) + return bundle + + +def verify_checkpoint_range_bundle( + bundle: SignedCheckpointRangeBundle, + trust: CheckpointPeerTrust, +) -> tuple[CheckpointRegistryRecord, ...]: + """Authenticate a peer and every exact record in one contiguous range.""" + bundle = SignedCheckpointRangeBundle.model_validate(bundle.model_dump(mode="json")) + trust = CheckpointPeerTrust.model_validate(trust.model_dump(mode="json")) + key = next( + ( + candidate + for candidate in trust.keys + if candidate.peer_id == bundle.statement.source_peer_id + and candidate.key_id == bundle.key_id + and candidate.status == "active" + ), + None, + ) + if key is None: + raise ValueError("checkpoint range signature does not use an active trusted key") + if bundle.statement.peer_trust_sha256 != trust.peer_trust_sha256: + raise ValueError("checkpoint range does not match peer trust") + expected = _checkpoint_range_statement( + bundle.records, + trust, + source_peer_id=bundle.statement.source_peer_id, + ) + if bundle.statement != expected: + raise ValueError("checkpoint range statement does not match records") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64(key.public_key_base64, label="peer public key", length=32) + ).verify( + _decode_base64( + bundle.signature_base64, + label="checkpoint range signature", + length=64, + ), + bundle.statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError("checkpoint range signature is invalid") from exc + return bundle.records + + +class CheckpointAcknowledgementStatement(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + source_peer_id: str = Field(min_length=1, max_length=200) + acknowledging_peer_id: str = Field(min_length=1, max_length=200) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + range_bundle_sha256: str = Field(pattern=_SHA256_PATTERN) + first_sequence: int = Field(ge=0) + acknowledged_record_sequence: int = Field(ge=0) + acknowledged_record_sha256: str = Field(pattern=_SHA256_PATTERN) + acknowledged_tree_size: int = Field(ge=2) + acknowledged_root_sha256: str = Field(pattern=_SHA256_PATTERN) + acknowledgement_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> CheckpointAcknowledgementStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.source_peer_id, label="source_peer_id") + _validate_identifier( + self.acknowledging_peer_id, + label="acknowledging_peer_id", + ) + if self.source_peer_id == self.acknowledging_peer_id: + raise ValueError("checkpoint acknowledgement requires a distinct peer") + if self.first_sequence > self.acknowledged_record_sequence: + raise ValueError("checkpoint acknowledgement range is inverted") + if self.acknowledgement_sha256 != _canonical_sha256( + self._binding_payload() + ): + raise ValueError("checkpoint acknowledgement fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"acknowledgement_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class CheckpointAcknowledgementSigningRequest(_FrozenModel): + acknowledging_peer_id: str + eligible_key_ids: tuple[str, ...] = Field(min_length=1) + statement: CheckpointAcknowledgementStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_request(self) -> CheckpointAcknowledgementSigningRequest: + if self.acknowledging_peer_id != self.statement.acknowledging_peer_id: + raise ValueError( + "checkpoint acknowledgement request peer does not match statement" + ) + if tuple(sorted(set(self.eligible_key_ids))) != self.eligible_key_ids: + raise ValueError("eligible acknowledgement key IDs must be sorted and unique") + expected = self.statement.signing_bytes() + decoded = _decode_base64( + self.signing_payload_base64, + label="checkpoint acknowledgement signing payload", + length=len(expected), + ) + if decoded != expected: + raise ValueError("checkpoint acknowledgement payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class SignedCheckpointAcknowledgement(_FrozenModel): + schema_version: int + statement: CheckpointAcknowledgementStatement + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + signed_acknowledgement_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_acknowledgement(self) -> SignedCheckpointAcknowledgement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.key_id, label="key_id") + _decode_base64( + self.signature_base64, + label="checkpoint acknowledgement signature", + length=64, + ) + if self.signed_acknowledgement_sha256 != _canonical_sha256( + self._binding_payload() + ): + raise ValueError("signed checkpoint acknowledgement fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump( + mode="json", + exclude={"signed_acknowledgement_sha256"}, + ) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_checkpoint_acknowledgement_request( + bundle: SignedCheckpointRangeBundle, + snapshot: CheckpointRegistrySnapshot, + trust: CheckpointPeerTrust, + *, + acknowledging_peer_id: str, +) -> CheckpointAcknowledgementSigningRequest: + """Confirm that an authenticated range is the receiver's exact current head.""" + records = verify_checkpoint_range_bundle(bundle, trust) + snapshot = CheckpointRegistrySnapshot.model_validate( + snapshot.model_dump(mode="json") + ) + if snapshot.registry_id != bundle.statement.registry_id: + raise ValueError("checkpoint acknowledgement snapshot uses another registry") + if snapshot.head_record_sha256 != records[-1].record_sha256: + raise ValueError("checkpoint acknowledgement range is not the registry head") + for record in records: + if record.sequence >= len(snapshot.records) or snapshot.records[record.sequence] != record: + raise ValueError("checkpoint acknowledgement range is not applied exactly") + eligible = tuple( + sorted( + key.key_id + for key in trust.keys + if key.peer_id == acknowledging_peer_id and key.status == "active" + ) + ) + if not eligible: + raise ValueError("acknowledging peer has no active trusted key") + last = records[-1] + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": bundle.statement.registry_id, + "source_peer_id": bundle.statement.source_peer_id, + "acknowledging_peer_id": acknowledging_peer_id, + "peer_trust_sha256": trust.peer_trust_sha256, + "range_bundle_sha256": bundle.range_bundle_sha256, + "first_sequence": records[0].sequence, + "acknowledged_record_sequence": last.sequence, + "acknowledged_record_sha256": last.record_sha256, + "acknowledged_tree_size": last.checkpoint.statement.current_tree_size, + "acknowledged_root_sha256": last.checkpoint.statement.current_root_sha256, + } + statement = CheckpointAcknowledgementStatement.model_validate( + {**payload, "acknowledgement_sha256": _canonical_sha256(payload)} + ) + return CheckpointAcknowledgementSigningRequest( + acknowledging_peer_id=acknowledging_peer_id, + eligible_key_ids=eligible, + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def build_signed_checkpoint_acknowledgement( + request: CheckpointAcknowledgementSigningRequest, + *, + key_id: str, + signature_base64: str, + peer_trust: CheckpointPeerTrust, +) -> SignedCheckpointAcknowledgement: + """Attach and verify one receiver signature over an applied range head.""" + request = CheckpointAcknowledgementSigningRequest.model_validate( + request.model_dump(mode="json") + ) + peer_trust = CheckpointPeerTrust.model_validate( + peer_trust.model_dump(mode="json") + ) + if request.statement.peer_trust_sha256 != peer_trust.peer_trust_sha256: + raise ValueError("checkpoint acknowledgement does not match peer trust") + active = { + key.key_id + for key in peer_trust.keys + if key.peer_id == request.acknowledging_peer_id and key.status == "active" + } + if request.eligible_key_ids != tuple(sorted(active)) or key_id not in active: + raise ValueError( + "checkpoint acknowledgement signature does not use an active trusted key" + ) + payload = { + "schema_version": SCHEMA_VERSION, + "statement": request.statement.model_dump(mode="json"), + "key_id": key_id, + "signature_base64": signature_base64, + } + acknowledgement = SignedCheckpointAcknowledgement.model_validate( + { + **payload, + "signed_acknowledgement_sha256": _canonical_sha256(payload), + } + ) + verify_checkpoint_acknowledgement(acknowledgement, peer_trust) + return acknowledgement + + +def verify_checkpoint_acknowledgement( + acknowledgement: SignedCheckpointAcknowledgement, + trust: CheckpointPeerTrust, +) -> CheckpointAcknowledgementStatement: + """Authenticate a receiver's exact range/head acknowledgement.""" + acknowledgement = SignedCheckpointAcknowledgement.model_validate( + acknowledgement.model_dump(mode="json") + ) + trust = CheckpointPeerTrust.model_validate(trust.model_dump(mode="json")) + statement = acknowledgement.statement + key = next( + ( + candidate + for candidate in trust.keys + if candidate.peer_id == statement.acknowledging_peer_id + and candidate.key_id == acknowledgement.key_id + and candidate.status == "active" + ), + None, + ) + if key is None: + raise ValueError( + "checkpoint acknowledgement signature does not use an active trusted key" + ) + if statement.peer_trust_sha256 != trust.peer_trust_sha256: + raise ValueError("checkpoint acknowledgement does not match peer trust") + if statement.registry_id != trust.registry_id: + raise ValueError("checkpoint acknowledgement does not match trust registry") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64(key.public_key_base64, label="peer public key", length=32) + ).verify( + _decode_base64( + acknowledgement.signature_base64, + label="checkpoint acknowledgement signature", + length=64, + ), + statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError("checkpoint acknowledgement signature is invalid") from exc + return statement + + +def _build_registry_record( + *, + registry_id: str, + sequence: int, + previous_record_sha256: str | None, + proof: TransparencyConsistencyProof, + checkpoint: SignedWitnessCheckpoint, + witness_trust: TransparencyWitnessTrust, + ledger: SignedAuthorityRootLedger, +) -> CheckpointRegistryRecord: + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": registry_id, + "sequence": sequence, + "previous_record_sha256": previous_record_sha256, + "authority_root_ledger_sha256": ledger.statement.ledger_sha256, + "witness_trust_sha256": witness_trust.witness_trust_sha256, + "consistency_proof": proof.model_dump(mode="json"), + "checkpoint": checkpoint.model_dump(mode="json"), + } + return CheckpointRegistryRecord.model_validate( + {**payload, "record_sha256": _canonical_sha256(payload)} + ) + + +def _validate_record_bindings( + record: CheckpointRegistryRecord, + witness_trust: TransparencyWitnessTrust, + ledger: SignedAuthorityRootLedger, +) -> None: + if record.authority_root_ledger_sha256 != ledger.statement.ledger_sha256: + raise ValueError("checkpoint registry record uses a different root ledger") + if record.witness_trust_sha256 != witness_trust.witness_trust_sha256: + raise ValueError("checkpoint registry record uses different witness trust") + verify_witness_checkpoint_bundle( + witness_trust, + record.consistency_proof, + ledger, + record.checkpoint, + ) + + +def _validate_registry_pair( + previous: CheckpointRegistryRecord, + current: CheckpointRegistryRecord, +) -> None: + detect_witness_checkpoint_conflict(previous.checkpoint, current.checkpoint) + old_current = previous.checkpoint.statement + new_previous = current.consistency_proof.previous_tree_head.statement + if ( + new_previous.tree_size, + new_previous.root_sha256, + new_previous.tree_head_sha256, + ) != ( + old_current.current_tree_size, + old_current.current_root_sha256, + old_current.current_tree_head_sha256, + ): + raise ValueError("checkpoint registry consistency chain is invalid") + + +class CheckpointRegistryStore: + """Locked, append-only JSONL storage for witnessed checkpoints.""" + + def __init__(self, path: str | Path, *, registry_id: str) -> None: + self.path = Path(path) + _validate_identifier(registry_id, label="registry_id") + self.registry_id = registry_id + + def _open_for_append(self) -> int: + self.path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open( + self.path, + os.O_CREAT + | os.O_RDWR + | os.O_APPEND + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + os.close(descriptor) + raise ValueError("checkpoint registry must be a regular file") + os.fchmod(descriptor, 0o600) + return descriptor + + def _open_for_replay(self) -> int | None: + try: + descriptor = os.open( + self.path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + except FileNotFoundError: + return None + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + os.close(descriptor) + raise ValueError("checkpoint registry must be a regular file") + return descriptor + + def _read_records( + self, + descriptor: int, + witness_trust: TransparencyWitnessTrust, + ledger: SignedAuthorityRootLedger, + ) -> tuple[CheckpointRegistryRecord, ...]: + size = os.fstat(descriptor).st_size + raw = os.pread(descriptor, size, 0) + if not raw: + return () + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("checkpoint registry must be UTF-8 JSONL") from exc + if not text.endswith("\n"): + raise ValueError("checkpoint registry has a truncated final record") + records: list[CheckpointRegistryRecord] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + try: + record = CheckpointRegistryRecord.model_validate_json(line) + except ValueError as exc: + raise ValueError( + f"invalid checkpoint registry record at line {line_number}: {exc}" + ) from exc + if record.registry_id != self.registry_id: + raise ValueError("checkpoint registry_id does not match store") + if record.sequence != len(records): + raise ValueError("checkpoint registry sequence is not contiguous") + expected_previous = records[-1].record_sha256 if records else None + if record.previous_record_sha256 != expected_previous: + raise ValueError("checkpoint registry hash chain is invalid") + _validate_record_bindings(record, witness_trust, ledger) + if records: + _validate_registry_pair(records[-1], record) + records.append(record) + return tuple(records) + + @staticmethod + def _append_bytes(descriptor: int, encoded: bytes, *, original_size: int) -> None: + position = 0 + try: + while position < len(encoded): + written = os.write(descriptor, encoded[position:]) + if written <= 0: + raise OSError("checkpoint registry append made no progress") + position += written + os.fsync(descriptor) + except BaseException: + os.ftruncate(descriptor, original_size) + os.fsync(descriptor) + raise + + def _snapshot( + self, + records: tuple[CheckpointRegistryRecord, ...], + ) -> CheckpointRegistrySnapshot: + last = records[-1] if records else None + return CheckpointRegistrySnapshot( + schema_version=SCHEMA_VERSION, + registry_id=self.registry_id, + records=records, + record_count=len(records), + head_record_sha256=last.record_sha256 if last else None, + current_tree_size=( + last.checkpoint.statement.current_tree_size if last else None + ), + current_root_sha256=( + last.checkpoint.statement.current_root_sha256 if last else None + ), + ) + + def replay( + self, + witness_trust: TransparencyWitnessTrust, + ledger: SignedAuthorityRootLedger, + ) -> CheckpointRegistrySnapshot: + """Replay and fully verify every durable record.""" + descriptor = self._open_for_replay() + if descriptor is None: + return self._snapshot(()) + try: + fcntl.flock(descriptor, fcntl.LOCK_SH) + records = self._read_records(descriptor, witness_trust, ledger) + return self._snapshot(records) + finally: + os.close(descriptor) + + def append( + self, + proof: TransparencyConsistencyProof, + checkpoint: SignedWitnessCheckpoint, + witness_trust: TransparencyWitnessTrust, + ledger: SignedAuthorityRootLedger, + *, + expected_record: CheckpointRegistryRecord | None = None, + ) -> CheckpointRegistryRecord: + """Verify and durably append one new checkpoint under an exclusive lock.""" + descriptor = self._open_for_append() + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + records = self._read_records(descriptor, witness_trust, ledger) + verify_witness_checkpoint_bundle(witness_trust, proof, ledger, checkpoint) + if records: + last = records[-1] + if ( + checkpoint.witness_checkpoint_sha256 + == last.checkpoint.witness_checkpoint_sha256 + and proof.consistency_proof_sha256 + == last.consistency_proof.consistency_proof_sha256 + ): + if expected_record is not None and expected_record != last: + raise ValueError("peer checkpoint record does not match local registry") + return last + detect_witness_checkpoint_conflict(last.checkpoint, checkpoint) + last_current = last.checkpoint.statement + incoming_previous = proof.previous_tree_head.statement + if incoming_previous.tree_size < last_current.current_tree_size: + raise ValueError("stale checkpoint cannot be appended") + if ( + incoming_previous.tree_size, + incoming_previous.root_sha256, + incoming_previous.tree_head_sha256, + ) != ( + last_current.current_tree_size, + last_current.current_root_sha256, + last_current.current_tree_head_sha256, + ): + raise ValueError("checkpoint does not extend the registry head") + record = _build_registry_record( + registry_id=self.registry_id, + sequence=len(records), + previous_record_sha256=(records[-1].record_sha256 if records else None), + proof=proof, + checkpoint=checkpoint, + witness_trust=witness_trust, + ledger=ledger, + ) + if expected_record is not None: + expected_record = CheckpointRegistryRecord.model_validate( + expected_record.model_dump(mode="json") + ) + if expected_record != record: + raise ValueError("peer checkpoint record does not match local registry") + encoded = (record.to_json() + "\n").encode() + self._append_bytes( + descriptor, + encoded, + original_size=os.fstat(descriptor).st_size, + ) + return record + finally: + os.close(descriptor) + + def import_packet( + self, + packet: SignedCheckpointExchangePacket, + peer_trust: CheckpointPeerTrust, + witness_trust: TransparencyWitnessTrust, + ledger: SignedAuthorityRootLedger, + ) -> CheckpointRegistryRecord: + """Authenticate, verify, and atomically import one exact peer record.""" + record = verify_checkpoint_exchange_packet(packet, peer_trust) + if peer_trust.registry_id != self.registry_id: + raise ValueError("peer trust registry_id does not match store") + return self.append( + record.consistency_proof, + record.checkpoint, + witness_trust, + ledger, + expected_record=record, + ) + + def import_range_bundle( + self, + bundle: SignedCheckpointRangeBundle, + peer_trust: CheckpointPeerTrust, + witness_trust: TransparencyWitnessTrust, + ledger: SignedAuthorityRootLedger, + ) -> CheckpointRegistrySnapshot: + """Atomically append the missing suffix of one authenticated range.""" + incoming = verify_checkpoint_range_bundle(bundle, peer_trust) + if peer_trust.registry_id != self.registry_id: + raise ValueError("peer trust registry_id does not match store") + for index, record in enumerate(incoming): + _validate_record_bindings(record, witness_trust, ledger) + if index: + _validate_registry_pair(incoming[index - 1], record) + + descriptor = self._open_for_append() + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + existing = self._read_records(descriptor, witness_trust, ledger) + first_sequence = incoming[0].sequence + if first_sequence > len(existing): + raise ValueError("checkpoint range would create a registry gap") + + overlap_end = min(len(existing), incoming[-1].sequence + 1) + for sequence in range(first_sequence, overlap_end): + received = incoming[sequence - first_sequence] + if existing[sequence] != received: + raise ValueError("conflicting range overlap detected") + + suffix_offset = max(0, len(existing) - first_sequence) + suffix = incoming[suffix_offset:] + if not suffix: + return self._snapshot(existing) + if suffix[0].sequence != len(existing): + raise ValueError("checkpoint range would create a registry gap") + expected_previous = existing[-1].record_sha256 if existing else None + if suffix[0].previous_record_sha256 != expected_previous: + raise ValueError("checkpoint range does not extend the registry head") + if existing: + _validate_registry_pair(existing[-1], suffix[0]) + + encoded = "".join(record.to_json() + "\n" for record in suffix).encode() + original_size = os.fstat(descriptor).st_size + self._append_bytes( + descriptor, + encoded, + original_size=original_size, + ) + return self._snapshot(existing + suffix) + finally: + os.close(descriptor) + + +class CheckpointPeerCursorRecord(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + cursor_sequence: int = Field(ge=0) + previous_cursor_record_sha256: str | None = Field( + default=None, + pattern=_SHA256_PATTERN, + ) + acknowledgement: SignedCheckpointAcknowledgement + cursor_record_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_record(self) -> CheckpointPeerCursorRecord: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + if self.cursor_record_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("peer cursor record fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"cursor_record_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointPeerCursorPosition(_FrozenModel): + source_peer_id: str + acknowledging_peer_id: str + acknowledged_record_sequence: int = Field(ge=0) + acknowledged_record_sha256: str = Field(pattern=_SHA256_PATTERN) + range_bundle_sha256: str = Field(pattern=_SHA256_PATTERN) + signed_acknowledgement_sha256: str = Field(pattern=_SHA256_PATTERN) + + +class CheckpointPeerCursorSnapshot(_FrozenModel): + schema_version: int + registry_id: str + cursors: tuple[CheckpointPeerCursorRecord, ...] + cursor_count: int = Field(ge=0) + head_cursor_record_sha256: str | None = Field( + default=None, + pattern=_SHA256_PATTERN, + ) + positions: tuple[CheckpointPeerCursorPosition, ...] + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _cursor_pair( + statement: CheckpointAcknowledgementStatement, +) -> tuple[str, str]: + return statement.source_peer_id, statement.acknowledging_peer_id + + +def _cursor_position( + acknowledgement: SignedCheckpointAcknowledgement, +) -> CheckpointPeerCursorPosition: + statement = acknowledgement.statement + return CheckpointPeerCursorPosition( + source_peer_id=statement.source_peer_id, + acknowledging_peer_id=statement.acknowledging_peer_id, + acknowledged_record_sequence=statement.acknowledged_record_sequence, + acknowledged_record_sha256=statement.acknowledged_record_sha256, + range_bundle_sha256=statement.range_bundle_sha256, + signed_acknowledgement_sha256=( + acknowledgement.signed_acknowledgement_sha256 + ), + ) + + +def _validate_cursor_advance( + current: CheckpointPeerCursorPosition | None, + acknowledgement: SignedCheckpointAcknowledgement, +) -> None: + if current is None: + return + statement = acknowledgement.statement + if statement.acknowledged_record_sequence < current.acknowledged_record_sequence: + raise ValueError("peer cursor regression is not allowed") + if statement.acknowledged_record_sequence == current.acknowledged_record_sequence: + if statement.acknowledged_record_sha256 != current.acknowledged_record_sha256: + raise ValueError("conflicting peer cursor acknowledgement detected") + raise ValueError("duplicate peer cursor acknowledgement record") + + +class CheckpointPeerCursorStore: + """Append-only acknowledgement ledger with monotonic per-peer cursors.""" + + def __init__(self, path: str | Path, *, registry_id: str) -> None: + self.path = Path(path) + _validate_identifier(registry_id, label="registry_id") + self.registry_id = registry_id + + def _open_for_append(self) -> int: + self.path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open( + self.path, + os.O_CREAT + | os.O_RDWR + | os.O_APPEND + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + os.close(descriptor) + raise ValueError("peer cursor ledger must be a regular file") + os.fchmod(descriptor, 0o600) + return descriptor + + def _open_for_replay(self) -> int | None: + try: + descriptor = os.open( + self.path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + except FileNotFoundError: + return None + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + os.close(descriptor) + raise ValueError("peer cursor ledger must be a regular file") + return descriptor + + def _read_records( + self, + descriptor: int, + trust: CheckpointPeerTrustSource, + ) -> tuple[CheckpointPeerCursorRecord, ...]: + size = os.fstat(descriptor).st_size + raw = os.pread(descriptor, size, 0) + if not raw: + return () + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("peer cursor ledger must be UTF-8 JSONL") from exc + if not text.endswith("\n"): + raise ValueError("peer cursor ledger has a truncated final record") + records: list[CheckpointPeerCursorRecord] = [] + positions: dict[tuple[str, str], CheckpointPeerCursorPosition] = {} + for line_number, line in enumerate(text.splitlines(), start=1): + try: + record = CheckpointPeerCursorRecord.model_validate_json(line) + except ValueError as exc: + raise ValueError( + f"invalid peer cursor record at line {line_number}: {exc}" + ) from exc + if record.registry_id != self.registry_id: + raise ValueError("peer cursor registry_id does not match store") + if record.cursor_sequence != len(records): + raise ValueError("peer cursor sequence is not contiguous") + expected_previous = records[-1].cursor_record_sha256 if records else None + if record.previous_cursor_record_sha256 != expected_previous: + raise ValueError("peer cursor hash chain is invalid") + resolved_trust = _resolve_peer_trust( + trust, + record.acknowledgement.statement.peer_trust_sha256, + ) + statement = verify_checkpoint_acknowledgement( + record.acknowledgement, + resolved_trust, + ) + pair = _cursor_pair(statement) + _validate_cursor_advance(positions.get(pair), record.acknowledgement) + positions[pair] = _cursor_position(record.acknowledgement) + records.append(record) + return tuple(records) + + def _snapshot( + self, + records: tuple[CheckpointPeerCursorRecord, ...], + ) -> CheckpointPeerCursorSnapshot: + latest: dict[tuple[str, str], CheckpointPeerCursorPosition] = {} + for record in records: + latest[_cursor_pair(record.acknowledgement.statement)] = _cursor_position( + record.acknowledgement + ) + return CheckpointPeerCursorSnapshot( + schema_version=SCHEMA_VERSION, + registry_id=self.registry_id, + cursors=records, + cursor_count=len(records), + head_cursor_record_sha256=( + records[-1].cursor_record_sha256 if records else None + ), + positions=tuple(latest[pair] for pair in sorted(latest)), + ) + + def replay( + self, + trust: CheckpointPeerTrustSource, + ) -> CheckpointPeerCursorSnapshot: + """Replay every acknowledgement signature and monotonic cursor transition.""" + descriptor = self._open_for_replay() + if descriptor is None: + return self._snapshot(()) + try: + fcntl.flock(descriptor, fcntl.LOCK_SH) + return self._snapshot(self._read_records(descriptor, trust)) + finally: + os.close(descriptor) + + def append( + self, + acknowledgement: SignedCheckpointAcknowledgement, + trust: CheckpointPeerTrustSource, + ) -> CheckpointPeerCursorRecord: + """Verify and append one monotonic peer acknowledgement.""" + resolved_trust = _resolve_peer_trust( + trust, + acknowledgement.statement.peer_trust_sha256, + ) + statement = verify_checkpoint_acknowledgement( + acknowledgement, + resolved_trust, + ) + if statement.registry_id != self.registry_id: + raise ValueError("checkpoint acknowledgement registry_id does not match store") + descriptor = self._open_for_append() + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + records = self._read_records(descriptor, trust) + for record in reversed(records): + if ( + record.acknowledgement.signed_acknowledgement_sha256 + == acknowledgement.signed_acknowledgement_sha256 + ): + return record + pair = _cursor_pair(statement) + current = next( + ( + _cursor_position(record.acknowledgement) + for record in reversed(records) + if _cursor_pair(record.acknowledgement.statement) == pair + ), + None, + ) + _validate_cursor_advance(current, acknowledgement) + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": self.registry_id, + "cursor_sequence": len(records), + "previous_cursor_record_sha256": ( + records[-1].cursor_record_sha256 if records else None + ), + "acknowledgement": acknowledgement.model_dump(mode="json"), + } + record = CheckpointPeerCursorRecord.model_validate( + {**payload, "cursor_record_sha256": _canonical_sha256(payload)} + ) + encoded = (record.to_json() + "\n").encode() + CheckpointRegistryStore._append_bytes( + descriptor, + encoded, + original_size=os.fstat(descriptor).st_size, + ) + return record + finally: + os.close(descriptor) diff --git a/benchmarks/agent_cli_comparison.py b/benchmarks/agent_cli_comparison.py new file mode 100644 index 0000000..8ac34bd --- /dev/null +++ b/benchmarks/agent_cli_comparison.py @@ -0,0 +1,290 @@ +"""Deterministic evaluation for recorded same-task agent CLI trials. + +This module never launches an agent engine. It validates observations captured +against one manifest and compares the three supported arms without inventing a +weighted composite score. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from enum import Enum +from statistics import median +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +SCHEMA_VERSION = 1 + + +class AgentCliArm(str, Enum): + """A supported arm in the same-task comparison.""" + + CODEX_CLI = "codex_cli" + CLAUDE_CODE = "claude_code" + MORPHIC_CONTROL = "morphic_control" + + +REQUIRED_ARMS = frozenset(AgentCliArm) + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +class BenchmarkTask(_FrozenModel): + """One immutable task definition shared by every trial.""" + + id: str = Field(min_length=1) + goal: str = Field(min_length=1) + workspace_revision: str = Field(min_length=1) + checks: tuple[str, ...] = Field(min_length=1) + handoff_assertions: tuple[str, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_checks(self) -> BenchmarkTask: + if any(not check.strip() for check in self.checks): + raise ValueError("task checks must not be blank") + if len(set(self.checks)) != len(self.checks): + raise ValueError("task checks must be unique") + if any(not assertion.strip() for assertion in self.handoff_assertions): + raise ValueError("task handoff_assertions must not be blank") + if len(set(self.handoff_assertions)) != len(self.handoff_assertions): + raise ValueError("task handoff_assertions must be unique") + return self + + +class AgentCliManifest(_FrozenModel): + """Reproducible contract for a same-task comparison.""" + + schema_version: int + benchmark_id: str = Field(min_length=1) + task: BenchmarkTask + arms: tuple[AgentCliArm, ...] + repetitions: int = Field(ge=1) + + @model_validator(mode="after") + def validate_contract(self) -> AgentCliManifest: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if len(self.arms) != len(REQUIRED_ARMS) or set(self.arms) != REQUIRED_ARMS: + expected = ", ".join(arm.value for arm in AgentCliArm) + raise ValueError(f"arms must contain exactly: {expected}") + return self + + +class TrialObservation(_FrozenModel): + """One recorded trial; all metrics are supplied by the recorder.""" + + arm: AgentCliArm + trial: int = Field(ge=1) + completed: bool + accepted_patch: bool + passed_checks: tuple[str, ...] + elapsed_seconds: float = Field(ge=0.0) + cost_usd: float = Field(ge=0.0) + human_interventions: int = Field(ge=0) + recovery_attempted: bool + recovery_succeeded: bool + passed_handoff_assertions: tuple[str, ...] + + @model_validator(mode="after") + def validate_outcomes(self) -> TrialObservation: + if self.accepted_patch and not self.completed: + raise ValueError("accepted_patch requires completed=true") + if self.recovery_succeeded and not self.recovery_attempted: + raise ValueError("recovery_succeeded requires recovery_attempted=true") + if len(set(self.passed_checks)) != len(self.passed_checks): + raise ValueError("passed_checks must be unique") + if len(set(self.passed_handoff_assertions)) != len(self.passed_handoff_assertions): + raise ValueError("passed_handoff_assertions must be unique") + return self + + +class RecordedResults(_FrozenModel): + """Recorded observations for one manifest.""" + + schema_version: int + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + observations: tuple[TrialObservation, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_schema_version(self) -> RecordedResults: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + return self + + +@dataclass(frozen=True) +class ArmMetrics: + """Unweighted metrics for one comparison arm.""" + + accepted_patch_rate: float + completion_rate: float + context_handoff_score: float + mean_cost_usd: float + mean_human_interventions: float + median_elapsed_seconds: float + recovery_rate: float | None + verification_rate: float + + +@dataclass(frozen=True) +class AgentCliReport: + """Deterministic same-task comparison report.""" + + schema_version: int + benchmark_id: str + task_id: str + workspace_revision: str + observation_count: int + arms: dict[str, ArmMetrics] + leaders: dict[str, list[str]] + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-safe report without timestamps or environment state.""" + return { + "schema_version": self.schema_version, + "benchmark_id": self.benchmark_id, + "task_id": self.task_id, + "workspace_revision": self.workspace_revision, + "observation_count": self.observation_count, + "arms": {name: asdict(metrics) for name, metrics in self.arms.items()}, + "leaders": self.leaders, + } + + def to_json(self) -> str: + """Serialize deterministically for diffs and CI artifacts.""" + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True) + + +def _round(value: float) -> float: + return round(value, 6) + + +def _calculate_metrics( + observations: list[TrialObservation], + check_count: int, + handoff_assertion_count: int, +) -> ArmMetrics: + count = len(observations) + recovery_trials = [row for row in observations if row.recovery_attempted] + recovery_rate = ( + _round(sum(row.recovery_succeeded for row in recovery_trials) / len(recovery_trials)) + if recovery_trials + else None + ) + return ArmMetrics( + accepted_patch_rate=_round(sum(row.accepted_patch for row in observations) / count), + completion_rate=_round(sum(row.completed for row in observations) / count), + context_handoff_score=_round( + sum(len(row.passed_handoff_assertions) for row in observations) + / (count * handoff_assertion_count) + ), + mean_cost_usd=_round(sum(row.cost_usd for row in observations) / count), + mean_human_interventions=_round( + sum(row.human_interventions for row in observations) / count + ), + median_elapsed_seconds=_round(median(row.elapsed_seconds for row in observations)), + recovery_rate=recovery_rate, + verification_rate=_round( + sum(len(row.passed_checks) for row in observations) / (count * check_count) + ), + ) + + +def _metric_leaders( + arms: dict[str, ArmMetrics], + metric: str, + *, + lower_is_better: bool, +) -> list[str]: + values = { + arm: value + for arm, metrics in arms.items() + if (value := getattr(metrics, metric)) is not None + } + if not values: + return [] + best = min(values.values()) if lower_is_better else max(values.values()) + return sorted(arm for arm, value in values.items() if value == best) + + +def evaluate_recorded_results( + manifest: AgentCliManifest, + results: RecordedResults, +) -> AgentCliReport: + """Validate and compare a complete recorded result set.""" + if results.benchmark_id != manifest.benchmark_id: + raise ValueError("results benchmark_id does not match manifest") + if results.task_id != manifest.task.id: + raise ValueError("results task_id does not match manifest") + + declared_checks = set(manifest.task.checks) + declared_handoff_assertions = set(manifest.task.handoff_assertions) + seen: set[tuple[AgentCliArm, int]] = set() + for observation in results.observations: + key = (observation.arm, observation.trial) + if key in seen: + raise ValueError( + f"duplicate observation for {observation.arm.value} trial {observation.trial}" + ) + seen.add(key) + unknown_checks = set(observation.passed_checks) - declared_checks + if unknown_checks: + names = ", ".join(sorted(unknown_checks)) + raise ValueError(f"observation contains undeclared checks: {names}") + unknown_assertions = ( + set(observation.passed_handoff_assertions) - declared_handoff_assertions + ) + if unknown_assertions: + names = ", ".join(sorted(unknown_assertions)) + raise ValueError(f"observation contains undeclared handoff assertions: {names}") + + expected = { + (arm, trial) + for arm in manifest.arms + for trial in range(1, manifest.repetitions + 1) + } + missing = expected - seen + extra = seen - expected + if missing: + details = ", ".join(f"{arm.value}:{trial}" for arm, trial in sorted(missing)) + raise ValueError(f"missing observations: {details}") + if extra: + details = ", ".join(f"{arm.value}:{trial}" for arm, trial in sorted(extra)) + raise ValueError(f"unexpected observations: {details}") + + arm_metrics = { + arm.value: _calculate_metrics( + [row for row in results.observations if row.arm == arm], + len(manifest.task.checks), + len(manifest.task.handoff_assertions), + ) + for arm in manifest.arms + } + lower_is_better = { + "accepted_patch_rate": False, + "completion_rate": False, + "context_handoff_score": False, + "mean_cost_usd": True, + "mean_human_interventions": True, + "median_elapsed_seconds": True, + "recovery_rate": False, + "verification_rate": False, + } + leaders = { + metric: _metric_leaders(arm_metrics, metric, lower_is_better=lower) + for metric, lower in lower_is_better.items() + } + return AgentCliReport( + schema_version=SCHEMA_VERSION, + benchmark_id=manifest.benchmark_id, + task_id=manifest.task.id, + workspace_revision=manifest.task.workspace_revision, + observation_count=len(results.observations), + arms=arm_metrics, + leaders=leaders, + ) diff --git a/benchmarks/agent_cli_gossip.py b/benchmarks/agent_cli_gossip.py new file mode 100644 index 0000000..d1df124 --- /dev/null +++ b/benchmarks/agent_cli_gossip.py @@ -0,0 +1,252 @@ +"""Signed checkpoint range and acknowledgement operations over local gossip.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Protocol + +from pydantic import BaseModel, ConfigDict, Field + +from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerCursorRecord, + CheckpointPeerCursorStore, + CheckpointPeerTrust, + CheckpointPeerTrustSource, + SignedCheckpointAcknowledgement, + SignedCheckpointRangeBundle, + verify_checkpoint_range_bundle, +) +from benchmarks.agent_cli_gossip_transport import ( + MAX_GOSSIP_RESPONSE_BYTES, + GossipRequestError, + send_checkpoint_gossip_request, +) + + +class CheckpointGossipRequestSender(Protocol): + async def __call__( + self, + *, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: ... + + +class _RequestModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +class _FetchRangeRequest(_RequestModel): + start_sequence: int = Field(ge=0) + max_records: int = Field(ge=1, le=1000) + + +class _SubmitAcknowledgementRequest(_RequestModel): + acknowledgement: SignedCheckpointAcknowledgement + + +def _resolve_peer_trust( + source: CheckpointPeerTrustSource, + peer_trust_sha256: str, +) -> CheckpointPeerTrust: + if isinstance(source, CheckpointPeerTrust): + if source.peer_trust_sha256 != peer_trust_sha256: + raise ValueError("checkpoint artifact does not match peer trust") + return source + resolved = source.resolve_peer_trust(peer_trust_sha256) + return CheckpointPeerTrust.model_validate(resolved.model_dump(mode="json")) + + +async def _send_gossip_request( + *, + descriptor_path: Path, + operation: str, + payload: dict[str, object], + request_sender: CheckpointGossipRequestSender | None, +) -> dict[str, object]: + if request_sender is None: + return await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation=operation, + payload=payload, + ) + return await request_sender(operation=operation, payload=payload) + + +class CheckpointGossipService: + """Expose pre-signed ranges and durably accept signed peer acknowledgements.""" + + def __init__( + self, + *, + registry_id: str, + source_peer_id: str, + range_bundles: tuple[SignedCheckpointRangeBundle, ...], + cursor_store: CheckpointPeerCursorStore, + peer_trust: CheckpointPeerTrustSource, + ) -> None: + if not registry_id or registry_id != registry_id.strip(): + raise ValueError("registry_id must be non-blank without surrounding whitespace") + if not source_peer_id or source_peer_id != source_peer_id.strip(): + raise ValueError( + "source_peer_id must be non-blank without surrounding whitespace" + ) + if cursor_store.registry_id != registry_id: + raise ValueError("checkpoint gossip cursor store uses another registry") + normalized = tuple( + SignedCheckpointRangeBundle.model_validate(bundle.model_dump(mode="json")) + for bundle in range_bundles + ) + if not normalized: + raise ValueError("checkpoint gossip requires at least one signed range bundle") + first_sequences: set[int] = set() + for bundle in normalized: + statement = bundle.statement + if statement.registry_id != registry_id: + raise ValueError("checkpoint gossip range uses another registry") + if statement.source_peer_id != source_peer_id: + raise ValueError("checkpoint gossip range uses another source peer") + if statement.first_sequence in first_sequences: + raise ValueError("checkpoint gossip range starts must be unique") + first_sequences.add(statement.first_sequence) + trust = _resolve_peer_trust(peer_trust, statement.peer_trust_sha256) + verify_checkpoint_range_bundle(bundle, trust) + encoded_size = len(bundle.to_json().encode("utf-8")) + if encoded_size > MAX_GOSSIP_RESPONSE_BYTES - 4096: + raise ValueError("checkpoint gossip range exceeds response limit") + self.registry_id = registry_id + self.source_peer_id = source_peer_id + self._range_bundles = tuple( + sorted(normalized, key=lambda item: item.statement.first_sequence) + ) + self._cursor_store = cursor_store + self._peer_trust = peer_trust + + async def dispatch( + self, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: + if operation == "status": + if payload: + raise GossipRequestError("invalid_status_payload") + return { + "available_ranges": [ + { + "first_sequence": bundle.statement.first_sequence, + "last_sequence": bundle.statement.last_sequence, + "range_bundle_sha256": bundle.range_bundle_sha256, + } + for bundle in self._range_bundles + ], + "registry_id": self.registry_id, + "source_peer_id": self.source_peer_id, + } + if operation == "fetch_range": + return self._fetch_range(payload) + if operation == "submit_acknowledgement": + return self._submit_acknowledgement(payload) + raise GossipRequestError("unsupported_operation") + + def _fetch_range(self, payload: dict[str, object]) -> dict[str, object]: + try: + request = _FetchRangeRequest.model_validate(payload) + except ValueError as exc: + raise GossipRequestError("invalid_range_request") from exc + bundle = next( + ( + candidate + for candidate in self._range_bundles + if candidate.statement.first_sequence <= request.start_sequence + <= candidate.statement.last_sequence + and len(candidate.records) <= request.max_records + ), + None, + ) + if bundle is None: + raise GossipRequestError("range_unavailable") + return {"range_bundle": bundle.model_dump(mode="json")} + + def _submit_acknowledgement( + self, + payload: dict[str, object], + ) -> dict[str, object]: + try: + request = _SubmitAcknowledgementRequest.model_validate(payload) + acknowledgement = request.acknowledgement + if acknowledgement.statement.registry_id != self.registry_id: + raise ValueError("checkpoint acknowledgement uses another registry") + if acknowledgement.statement.source_peer_id != self.source_peer_id: + raise ValueError("checkpoint acknowledgement uses another source peer") + record = self._cursor_store.append( + acknowledgement, + self._peer_trust, + ) + except (OSError, ValueError) as exc: + raise GossipRequestError("invalid_acknowledgement") from exc + return {"cursor_record": record.model_dump(mode="json")} + + +async def fetch_signed_checkpoint_range( + *, + descriptor_path: Path, + start_sequence: int, + max_records: int, + peer_trust: CheckpointPeerTrustSource, + request_sender: CheckpointGossipRequestSender | None = None, +) -> SignedCheckpointRangeBundle: + """Fetch and independently verify one already signed range bundle.""" + request = _FetchRangeRequest( + start_sequence=start_sequence, + max_records=max_records, + ) + result = await _send_gossip_request( + descriptor_path=descriptor_path, + operation="fetch_range", + payload=request.model_dump(mode="json"), + request_sender=request_sender, + ) + payload = result.get("range_bundle") + if not isinstance(payload, dict): + raise RuntimeError("checkpoint gossip response omitted its range bundle") + bundle = SignedCheckpointRangeBundle.model_validate(payload) + trust = _resolve_peer_trust(peer_trust, bundle.statement.peer_trust_sha256) + verify_checkpoint_range_bundle(bundle, trust) + return bundle + + +async def submit_signed_checkpoint_acknowledgement( + *, + descriptor_path: Path, + acknowledgement: SignedCheckpointAcknowledgement, + request_sender: CheckpointGossipRequestSender | None = None, +) -> CheckpointPeerCursorRecord: + """Submit one signed acknowledgement and validate the returned cursor record.""" + request = _SubmitAcknowledgementRequest(acknowledgement=acknowledgement) + result = await _send_gossip_request( + descriptor_path=descriptor_path, + operation="submit_acknowledgement", + payload=request.model_dump(mode="json"), + request_sender=request_sender, + ) + payload = result.get("cursor_record") + if not isinstance(payload, dict): + raise RuntimeError("checkpoint gossip response omitted its cursor record") + record = CheckpointPeerCursorRecord.model_validate(payload) + if record.acknowledgement != acknowledgement: + raise RuntimeError("checkpoint gossip cursor response changed acknowledgement") + return record + + +async def fetch_checkpoint_gossip_status( + *, + descriptor_path: Path, + request_sender: CheckpointGossipRequestSender | None = None, +) -> dict[str, object]: + """Return authenticated online range availability without mutating either peer.""" + return await _send_gossip_request( + descriptor_path=descriptor_path, + operation="status", + payload={}, + request_sender=request_sender, + ) diff --git a/benchmarks/agent_cli_gossip_sync.py b/benchmarks/agent_cli_gossip_sync.py new file mode 100644 index 0000000..20e6cd8 --- /dev/null +++ b/benchmarks/agent_cli_gossip_sync.py @@ -0,0 +1,730 @@ +"""Bounded resumable checkpoint catch-up with durable trust-pinned audit.""" + +from __future__ import annotations + +import asyncio +import fcntl +import hashlib +import json +import os +import stat +from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager +from functools import partial +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointRegistrySnapshot, + CheckpointRegistryStore, + SignedCheckpointRangeBundle, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_gossip import ( + CheckpointGossipRequestSender, + fetch_checkpoint_gossip_status, + fetch_signed_checkpoint_range, +) +from benchmarks.agent_cli_peer_trust_ledger import CheckpointPeerTrustLedger +from benchmarks.agent_cli_transparency import SignedAuthorityRootLedger +from benchmarks.agent_cli_witness import TransparencyWitnessTrust + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" +_SyncEvent = Literal["imported", "recovered", "retry", "stopped", "trust_advanced"] +_StopReason = Literal[ + "up_to_date", + "range_gap", + "record_budget_exhausted", + "round_budget_exhausted", + "retry_exhausted", +] +_TransportOperation = Literal["status", "fetch_range"] + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if not identifier or identifier != identifier.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +class CheckpointGossipSyncPolicy(_FrozenModel): + max_rounds: int = Field(default=16, ge=1, le=100) + max_records: int = Field(default=1000, ge=1, le=10_000) + max_attempts_per_request: int = Field(default=3, ge=1, le=10) + retry_delays_seconds: tuple[float, ...] = (0.05, 0.1) + + @model_validator(mode="after") + def validate_policy(self) -> CheckpointGossipSyncPolicy: + expected = self.max_attempts_per_request - 1 + if len(self.retry_delays_seconds) != expected: + raise ValueError("retry delay count must be max attempts minus one") + if any(delay < 0 or delay > 5 for delay in self.retry_delays_seconds): + raise ValueError("retry delays must be between zero and five seconds") + if sum(self.retry_delays_seconds) > 30: + raise ValueError("total retry delay must not exceed 30 seconds") + return self + + @property + def policy_sha256(self) -> str: + return _canonical_sha256(self.model_dump(mode="json")) + + +class CheckpointGossipSyncAuditRecord(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + source_peer_id: str = Field(min_length=1, max_length=200) + audit_sequence: int = Field(ge=0) + previous_record_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + event: _SyncEvent + peer_trust_generation: int = Field(ge=1) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + peer_trust_ledger_sha256: str = Field(pattern=_SHA256_PATTERN) + sync_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + local_record_count: int = Field(ge=0) + local_head_record_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + range_bundle_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + first_sequence: int | None = Field(default=None, ge=0) + last_sequence: int | None = Field(default=None, ge=0) + imported_records: int = Field(default=0, ge=0) + transport_operation: _TransportOperation | None = None + transport_attempt: int | None = Field(default=None, ge=1, le=10) + stop_reason: _StopReason | None = None + record_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_record(self) -> CheckpointGossipSyncAuditRecord: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.source_peer_id, label="source_peer_id") + if (self.local_record_count == 0) != ( + self.local_head_record_sha256 is None + ): + raise ValueError("sync audit local head does not match record count") + range_fields = ( + self.range_bundle_sha256, + self.first_sequence, + self.last_sequence, + ) + if self.event == "imported": + if any(value is None for value in range_fields) or self.imported_records < 1: + raise ValueError("imported sync audit record is incomplete") + assert self.first_sequence is not None + assert self.last_sequence is not None + if self.first_sequence > self.last_sequence: + raise ValueError("imported sync audit range is inverted") + elif any(value is not None for value in range_fields) or self.imported_records: + raise ValueError("non-import sync audit record carries range metadata") + if self.event == "retry": + if self.transport_operation is None or self.transport_attempt is None: + raise ValueError("retry sync audit record is incomplete") + elif self.transport_operation is not None or self.transport_attempt is not None: + raise ValueError("non-retry sync audit record carries retry metadata") + if (self.event == "stopped") != (self.stop_reason is not None): + raise ValueError("sync audit stop reason does not match event") + if self.record_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("sync audit record fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"record_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointGossipSyncAuditSnapshot(_FrozenModel): + schema_version: int + registry_id: str + source_peer_id: str + records: tuple[CheckpointGossipSyncAuditRecord, ...] + record_count: int = Field(ge=0) + head_record_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + local_record_count: int = Field(ge=0) + local_head_record_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + peer_trust_generation: int | None = Field(default=None, ge=1) + peer_trust_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_snapshot(self) -> CheckpointGossipSyncAuditSnapshot: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.record_count != len(self.records): + raise ValueError("sync audit snapshot record count does not match") + last = self.records[-1] if self.records else None + expected = ( + last.record_sha256 if last else None, + last.local_record_count if last else 0, + last.local_head_record_sha256 if last else None, + last.peer_trust_generation if last else None, + last.peer_trust_sha256 if last else None, + ) + actual = ( + self.head_record_sha256, + self.local_record_count, + self.local_head_record_sha256, + self.peer_trust_generation, + self.peer_trust_sha256, + ) + if actual != expected: + raise ValueError("sync audit snapshot head metadata does not match records") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointGossipSyncResult(_FrozenModel): + schema_version: int + registry_id: str + source_peer_id: str + stop_reason: _StopReason + rounds_attempted: int = Field(ge=0) + ranges_imported: int = Field(ge=0) + records_imported: int = Field(ge=0) + retries: int = Field(ge=0) + local_record_count: int = Field(ge=0) + local_head_record_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + peer_trust_generation: int = Field(ge=1) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + sync_policy_sha256: str = Field(pattern=_SHA256_PATTERN) + sync_audit_head_sha256: str = Field(pattern=_SHA256_PATTERN) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class _AvailableRange(_FrozenModel): + first_sequence: int = Field(ge=0) + last_sequence: int = Field(ge=0) + range_bundle_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_range(self) -> _AvailableRange: + if self.first_sequence > self.last_sequence: + raise ValueError("checkpoint gossip status range is inverted") + return self + + +class _GossipStatus(_FrozenModel): + registry_id: str + source_peer_id: str + available_ranges: tuple[_AvailableRange, ...] + + @model_validator(mode="after") + def validate_status(self) -> _GossipStatus: + starts = [item.first_sequence for item in self.available_ranges] + if starts != sorted(starts) or len(starts) != len(set(starts)): + raise ValueError("checkpoint gossip status ranges must be sorted and unique") + return self + + +class CheckpointGossipSyncAuditStore: + """Single-writer append-only audit for one registry/source pull loop.""" + + def __init__( + self, + path: str | Path, + *, + registry_id: str, + source_peer_id: str, + ) -> None: + self.path = Path(path) + _validate_identifier(registry_id, label="registry_id") + _validate_identifier(source_peer_id, label="source_peer_id") + self.registry_id = registry_id + self.source_peer_id = source_peer_id + + def _open(self, *, create: bool) -> int | None: + flags = ( + os.O_RDWR if create else os.O_RDONLY + ) | getattr(os, "O_NOFOLLOW", 0) + if create: + self.path.parent.mkdir(parents=True, exist_ok=True) + flags |= os.O_CREAT | os.O_APPEND + try: + descriptor = os.open(self.path, flags, 0o600) + except FileNotFoundError: + return None + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + os.close(descriptor) + raise ValueError("sync audit ledger must be a regular file") + if create: + os.fchmod(descriptor, 0o600) + return descriptor + + @contextmanager + def locked(self) -> Iterator[_CheckpointGossipSyncAuditWriter]: + descriptor = self._open(create=True) + assert descriptor is not None + try: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise RuntimeError("checkpoint gossip sync is already running") from exc + yield _CheckpointGossipSyncAuditWriter(self, descriptor) + finally: + os.close(descriptor) + + def replay( + self, + peer_trust_ledger: CheckpointPeerTrustLedger, + ) -> CheckpointGossipSyncAuditSnapshot: + descriptor = self._open(create=False) + if descriptor is None: + return self._snapshot(()) + try: + fcntl.flock(descriptor, fcntl.LOCK_SH) + records = self._read_records(descriptor, peer_trust_ledger) + return self._snapshot(records) + finally: + os.close(descriptor) + + def _read_records( + self, + descriptor: int, + peer_trust_ledger: CheckpointPeerTrustLedger, + ) -> tuple[CheckpointGossipSyncAuditRecord, ...]: + ledger = CheckpointPeerTrustLedger.model_validate( + peer_trust_ledger.model_dump(mode="json") + ) + size = os.fstat(descriptor).st_size + raw = os.pread(descriptor, size, 0) + if not raw: + return () + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("sync audit ledger must be UTF-8 JSONL") from exc + if not text.endswith("\n"): + raise ValueError("sync audit ledger has a truncated final record") + records: list[CheckpointGossipSyncAuditRecord] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + try: + record = CheckpointGossipSyncAuditRecord.model_validate_json(line) + except ValueError as exc: + raise ValueError( + f"invalid sync audit record at line {line_number}: {exc}" + ) from exc + if ( + record.registry_id != self.registry_id + or record.source_peer_id != self.source_peer_id + ): + raise ValueError("sync audit record endpoint does not match store") + if record.audit_sequence != len(records): + raise ValueError("sync audit sequence is not contiguous") + expected_previous = records[-1].record_sha256 if records else None + if record.previous_record_sha256 != expected_previous: + raise ValueError("sync audit hash chain is invalid") + if records: + previous = records[-1] + if record.local_record_count < previous.local_record_count: + raise ValueError("sync audit local record count regressed") + if record.peer_trust_generation < previous.peer_trust_generation: + raise ValueError("sync audit peer trust generation regressed") + if record.event == "imported": + if record.local_record_count != ( + previous.local_record_count + record.imported_records + ): + raise ValueError("sync audit imported record count does not advance") + elif record.event != "recovered" and ( + record.local_record_count != previous.local_record_count + or record.local_head_record_sha256 + != previous.local_head_record_sha256 + ): + raise ValueError("sync audit non-import event changes registry head") + if record.peer_trust_generation > ledger.active_generation: + raise ValueError("peer trust ledger rollback detected by sync audit pin") + pinned_generation = ledger.generations[record.peer_trust_generation - 1] + if pinned_generation.trust.peer_trust_sha256 != record.peer_trust_sha256: + raise ValueError("peer trust ledger fork detected at sync audit pin") + if ( + record.peer_trust_generation == ledger.active_generation + and record.peer_trust_ledger_sha256 != ledger.ledger_sha256 + ): + raise ValueError("peer trust ledger fork detected at active generation") + records.append(record) + if records and ledger.active_generation < records[-1].peer_trust_generation: + raise ValueError("peer trust ledger rollback detected by sync audit pin") + return tuple(records) + + def _snapshot( + self, + records: tuple[CheckpointGossipSyncAuditRecord, ...], + ) -> CheckpointGossipSyncAuditSnapshot: + last = records[-1] if records else None + return CheckpointGossipSyncAuditSnapshot( + schema_version=SCHEMA_VERSION, + registry_id=self.registry_id, + source_peer_id=self.source_peer_id, + records=records, + record_count=len(records), + head_record_sha256=last.record_sha256 if last else None, + local_record_count=last.local_record_count if last else 0, + local_head_record_sha256=( + last.local_head_record_sha256 if last else None + ), + peer_trust_generation=(last.peer_trust_generation if last else None), + peer_trust_sha256=(last.peer_trust_sha256 if last else None), + ) + + +class _CheckpointGossipSyncAuditWriter: + def __init__( + self, + store: CheckpointGossipSyncAuditStore, + descriptor: int, + ) -> None: + self._store = store + self._descriptor = descriptor + + def replay( + self, + peer_trust_ledger: CheckpointPeerTrustLedger, + ) -> CheckpointGossipSyncAuditSnapshot: + return self._store._snapshot( + self._store._read_records(self._descriptor, peer_trust_ledger) + ) + + def append( + self, + *, + peer_trust_ledger: CheckpointPeerTrustLedger, + registry_snapshot: CheckpointRegistrySnapshot, + sync_policy_sha256: str, + event: _SyncEvent, + range_bundle_sha256: str | None = None, + first_sequence: int | None = None, + last_sequence: int | None = None, + imported_records: int = 0, + transport_operation: _TransportOperation | None = None, + transport_attempt: int | None = None, + stop_reason: _StopReason | None = None, + ) -> CheckpointGossipSyncAuditRecord: + records = self._store._read_records(self._descriptor, peer_trust_ledger) + active = peer_trust_ledger.active_trust + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": self._store.registry_id, + "source_peer_id": self._store.source_peer_id, + "audit_sequence": len(records), + "previous_record_sha256": ( + records[-1].record_sha256 if records else None + ), + "event": event, + "peer_trust_generation": peer_trust_ledger.active_generation, + "peer_trust_sha256": active.peer_trust_sha256, + "peer_trust_ledger_sha256": peer_trust_ledger.ledger_sha256, + "sync_policy_sha256": sync_policy_sha256, + "local_record_count": registry_snapshot.record_count, + "local_head_record_sha256": registry_snapshot.head_record_sha256, + "range_bundle_sha256": range_bundle_sha256, + "first_sequence": first_sequence, + "last_sequence": last_sequence, + "imported_records": imported_records, + "transport_operation": transport_operation, + "transport_attempt": transport_attempt, + "stop_reason": stop_reason, + } + record = CheckpointGossipSyncAuditRecord.model_validate( + {**payload, "record_sha256": _canonical_sha256(payload)} + ) + encoded = (record.to_json() + "\n").encode("utf-8") + CheckpointRegistryStore._append_bytes( + self._descriptor, + encoded, + original_size=os.fstat(self._descriptor).st_size, + ) + return record + + +async def run_checkpoint_gossip_sync( + *, + descriptor_path: Path, + registry_store: CheckpointRegistryStore, + audit_store: CheckpointGossipSyncAuditStore, + peer_trust_ledger: CheckpointPeerTrustLedger, + witness_trust: TransparencyWitnessTrust, + authority_root_ledger: SignedAuthorityRootLedger, + policy: CheckpointGossipSyncPolicy, + request_sender: CheckpointGossipRequestSender | None = None, +) -> CheckpointGossipSyncResult: + """Pull exact signed ranges until current, bounded, or safely stopped.""" + ledger = CheckpointPeerTrustLedger.model_validate( + peer_trust_ledger.model_dump(mode="json") + ) + policy = CheckpointGossipSyncPolicy.model_validate(policy.model_dump(mode="json")) + if registry_store.registry_id != audit_store.registry_id: + raise ValueError("checkpoint gossip sync registry and audit stores differ") + if ledger.registry_id != registry_store.registry_id: + raise ValueError("checkpoint gossip sync peer trust uses another registry") + rounds = 0 + ranges_imported = 0 + records_imported = 0 + retries = 0 + + with audit_store.locked() as audit: + snapshot = registry_store.replay(witness_trust, authority_root_ledger) + audit_snapshot = audit.replay(ledger) + _validate_audit_against_registry(audit_snapshot, snapshot) + if audit_snapshot.local_record_count < snapshot.record_count: + audit.append( + peer_trust_ledger=ledger, + registry_snapshot=snapshot, + sync_policy_sha256=policy.policy_sha256, + event="recovered", + ) + audit_snapshot = audit.replay(ledger) + if ( + audit_snapshot.peer_trust_generation is not None + and audit_snapshot.peer_trust_generation < ledger.active_generation + ): + audit.append( + peer_trust_ledger=ledger, + registry_snapshot=snapshot, + sync_policy_sha256=policy.policy_sha256, + event="trust_advanced", + ) + + async def request_with_retry( + operation: _TransportOperation, + request: Callable[[], Awaitable[object]], + ) -> object | None: + nonlocal retries + for attempt in range(1, policy.max_attempts_per_request + 1): + try: + return await request() + except (OSError, RuntimeError, TimeoutError): + if attempt >= policy.max_attempts_per_request: + return None + retries += 1 + audit.append( + peer_trust_ledger=ledger, + registry_snapshot=snapshot, + sync_policy_sha256=policy.policy_sha256, + event="retry", + transport_operation=operation, + transport_attempt=attempt, + ) + await asyncio.sleep(policy.retry_delays_seconds[attempt - 1]) + return None + + async def fetch_status() -> dict[str, object]: + if request_sender is None: + return await fetch_checkpoint_gossip_status( + descriptor_path=descriptor_path + ) + return await fetch_checkpoint_gossip_status( + descriptor_path=descriptor_path, + request_sender=request_sender, + ) + + while rounds < policy.max_rounds: + raw_status = await request_with_retry( + "status", + fetch_status, + ) + if raw_status is None: + return _stop_sync( + audit, + ledger, + snapshot, + audit_store, + policy, + reason="retry_exhausted", + rounds=rounds, + ranges_imported=ranges_imported, + records_imported=records_imported, + retries=retries, + ) + status = _GossipStatus.model_validate(raw_status) + if ( + status.registry_id != registry_store.registry_id + or status.source_peer_id != audit_store.source_peer_id + ): + raise ValueError("checkpoint gossip status endpoint does not match sync") + candidate = _select_range(status, snapshot.record_count) + if candidate is None: + reason: _StopReason = ( + "range_gap" + if any( + item.first_sequence > snapshot.record_count + for item in status.available_ranges + ) + else "up_to_date" + ) + return _stop_sync( + audit, + ledger, + snapshot, + audit_store, + policy, + reason=reason, + rounds=rounds, + ranges_imported=ranges_imported, + records_imported=records_imported, + retries=retries, + ) + new_records = candidate.last_sequence - snapshot.record_count + 1 + if records_imported + new_records > policy.max_records: + return _stop_sync( + audit, + ledger, + snapshot, + audit_store, + policy, + reason="record_budget_exhausted", + rounds=rounds, + ranges_imported=ranges_imported, + records_imported=records_imported, + retries=retries, + ) + rounds += 1 + next_sequence = snapshot.record_count + bundle_record_count = ( + candidate.last_sequence - candidate.first_sequence + 1 + ) + fetch_request: Callable[[], Awaitable[object]] = partial( + fetch_signed_checkpoint_range, + descriptor_path=descriptor_path, + start_sequence=next_sequence, + max_records=bundle_record_count, + peer_trust=ledger, + request_sender=request_sender, + ) + bundle_result = await request_with_retry( + "fetch_range", + fetch_request, + ) + if bundle_result is None: + return _stop_sync( + audit, + ledger, + snapshot, + audit_store, + policy, + reason="retry_exhausted", + rounds=rounds, + ranges_imported=ranges_imported, + records_imported=records_imported, + retries=retries, + ) + bundle = SignedCheckpointRangeBundle.model_validate(bundle_result) + if bundle.range_bundle_sha256 != candidate.range_bundle_sha256: + raise ValueError("checkpoint gossip status changed before range fetch") + resolved_trust = ledger.resolve_peer_trust( + bundle.statement.peer_trust_sha256 + ) + previous_count = snapshot.record_count + snapshot = registry_store.import_range_bundle( + bundle, + resolved_trust, + witness_trust, + authority_root_ledger, + ) + imported = snapshot.record_count - previous_count + if imported != new_records or imported < 1: + raise ValueError("checkpoint gossip range made unexpected progress") + ranges_imported += 1 + records_imported += imported + audit.append( + peer_trust_ledger=ledger, + registry_snapshot=snapshot, + sync_policy_sha256=policy.policy_sha256, + event="imported", + range_bundle_sha256=bundle.range_bundle_sha256, + first_sequence=bundle.statement.first_sequence, + last_sequence=bundle.statement.last_sequence, + imported_records=imported, + ) + + return _stop_sync( + audit, + ledger, + snapshot, + audit_store, + policy, + reason="round_budget_exhausted", + rounds=rounds, + ranges_imported=ranges_imported, + records_imported=records_imported, + retries=retries, + ) + + +def _validate_audit_against_registry( + audit: CheckpointGossipSyncAuditSnapshot, + registry: CheckpointRegistrySnapshot, +) -> None: + if audit.local_record_count > registry.record_count: + raise ValueError("sync audit is ahead of verified checkpoint registry") + if audit.local_record_count == 0: + return + expected_head = registry.records[audit.local_record_count - 1].record_sha256 + if audit.local_head_record_sha256 != expected_head: + raise ValueError("sync audit pinned registry history does not match") + + +def _select_range(status: _GossipStatus, next_sequence: int) -> _AvailableRange | None: + candidates = tuple( + item + for item in status.available_ranges + if item.first_sequence <= next_sequence <= item.last_sequence + ) + return max(candidates, key=lambda item: item.first_sequence, default=None) + + +def _stop_sync( + audit: _CheckpointGossipSyncAuditWriter, + ledger: CheckpointPeerTrustLedger, + snapshot: CheckpointRegistrySnapshot, + audit_store: CheckpointGossipSyncAuditStore, + policy: CheckpointGossipSyncPolicy, + *, + reason: _StopReason, + rounds: int, + ranges_imported: int, + records_imported: int, + retries: int, +) -> CheckpointGossipSyncResult: + record = audit.append( + peer_trust_ledger=ledger, + registry_snapshot=snapshot, + sync_policy_sha256=policy.policy_sha256, + event="stopped", + stop_reason=reason, + ) + return CheckpointGossipSyncResult( + schema_version=SCHEMA_VERSION, + registry_id=audit_store.registry_id, + source_peer_id=audit_store.source_peer_id, + stop_reason=reason, + rounds_attempted=rounds, + ranges_imported=ranges_imported, + records_imported=records_imported, + retries=retries, + local_record_count=snapshot.record_count, + local_head_record_sha256=snapshot.head_record_sha256, + peer_trust_generation=ledger.active_generation, + peer_trust_sha256=ledger.active_trust.peer_trust_sha256, + sync_policy_sha256=policy.policy_sha256, + sync_audit_head_sha256=record.record_sha256, + ) diff --git a/benchmarks/agent_cli_gossip_tls_identity.py b/benchmarks/agent_cli_gossip_tls_identity.py new file mode 100644 index 0000000..1649d85 --- /dev/null +++ b/benchmarks/agent_cli_gossip_tls_identity.py @@ -0,0 +1,650 @@ +"""Peer-signed TLS certificate enrollment for checkpoint gossip.""" + +from __future__ import annotations + +import base64 +import hashlib +import ipaddress +import json +from datetime import UTC, datetime +from typing import Literal + +from cryptography import x509 +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from cryptography.x509.oid import ExtendedKeyUsageOID +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerKey, + CheckpointPeerTrust, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode()).hexdigest() + + +def _validate_identifier(value: str, *, label: str) -> None: + if not value or value != value.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +def _load_certificate(certificate: bytes) -> x509.Certificate: + try: + if b"-----BEGIN CERTIFICATE-----" in certificate: + certificates = x509.load_pem_x509_certificates(certificate) + if len(certificates) != 1: + raise ValueError("TLS enrollment must contain exactly one certificate") + return certificates[0] + return x509.load_der_x509_certificate(certificate) + except ValueError as exc: + raise ValueError("TLS enrollment certificate is invalid") from exc + + +def _certificate_metadata(certificate_bytes: bytes) -> dict[str, object]: + certificate = _load_certificate(certificate_bytes) + try: + constraints = certificate.extensions.get_extension_for_class(x509.BasicConstraints).value + san = certificate.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + usages = certificate.extensions.get_extension_for_class(x509.ExtendedKeyUsage).value + key_usage = certificate.extensions.get_extension_for_class(x509.KeyUsage).value + except x509.ExtensionNotFound as exc: + raise ValueError("TLS leaf certificate is missing required extensions") from exc + if constraints.ca: + raise ValueError("TLS enrollment certificate must be a leaf certificate") + required = {ExtendedKeyUsageOID.CLIENT_AUTH, ExtendedKeyUsageOID.SERVER_AUTH} + if not required.issubset(set(usages)): + raise ValueError("TLS leaf certificate must allow client and server authentication") + if not key_usage.digital_signature: + raise ValueError("TLS leaf certificate must allow digital signatures") + dns_names = tuple(sorted(set(san.get_values_for_type(x509.DNSName)))) + ip_addresses = tuple( + sorted( + {str(value) for value in san.get_values_for_type(x509.IPAddress)}, + key=ipaddress.ip_address, + ) + ) + if not dns_names and not ip_addresses: + raise ValueError("TLS leaf certificate must declare a DNS or IP SAN") + der = certificate.public_bytes(serialization.Encoding.DER) + spki = certificate.public_key().public_bytes( + serialization.Encoding.DER, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + return { + "certificate_sha256": hashlib.sha256(der).hexdigest(), + "spki_sha256": hashlib.sha256(spki).hexdigest(), + "subject": certificate.subject.rfc4514_string(), + "issuer": certificate.issuer.rfc4514_string(), + "serial_number": format(certificate.serial_number, "x"), + "not_valid_before": certificate.not_valid_before_utc.astimezone(UTC).isoformat(), + "not_valid_after": certificate.not_valid_after_utc.astimezone(UTC).isoformat(), + "dns_names": dns_names, + "ip_addresses": ip_addresses, + "extended_key_usages": ("client_auth", "server_auth"), + } + + +class CheckpointPeerTlsEnrollmentStatement(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + peer_id: str = Field(min_length=1, max_length=200) + generation: int = Field(ge=1) + previous_enrollment_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + certificate_sha256: str = Field(pattern=_SHA256_PATTERN) + spki_sha256: str = Field(pattern=_SHA256_PATTERN) + subject: str = Field(min_length=1) + issuer: str = Field(min_length=1) + serial_number: str = Field(min_length=1) + not_valid_before: str = Field(min_length=1) + not_valid_after: str = Field(min_length=1) + dns_names: tuple[str, ...] + ip_addresses: tuple[str, ...] + extended_key_usages: tuple[Literal["client_auth", "server_auth"], ...] + statement_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> CheckpointPeerTlsEnrollmentStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.peer_id, label="peer_id") + if (self.generation == 1) != (self.previous_enrollment_sha256 is None): + raise ValueError("TLS enrollment generation and predecessor do not match") + if tuple(sorted(set(self.dns_names))) != self.dns_names: + raise ValueError("TLS enrollment DNS names must be sorted and unique") + if tuple(sorted(set(self.ip_addresses), key=ipaddress.ip_address)) != self.ip_addresses: + raise ValueError("TLS enrollment IP addresses must be sorted and unique") + if self.extended_key_usages != ("client_auth", "server_auth"): + raise ValueError("TLS enrollment usages must bind client and server authentication") + if self.statement_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("TLS enrollment statement fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"statement_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class CheckpointPeerTlsEnrollmentTemplate(_FrozenModel): + statement: CheckpointPeerTlsEnrollmentStatement + eligible_key_ids: tuple[str, ...] = Field(min_length=1) + signing_payload_base64: str = Field(min_length=1) + signatures_completed: Literal[False] = False + + @model_validator(mode="after") + def validate_template(self) -> CheckpointPeerTlsEnrollmentTemplate: + if tuple(sorted(set(self.eligible_key_ids))) != self.eligible_key_ids: + raise ValueError("eligible peer key IDs must be sorted and unique") + expected = base64.b64encode(self.statement.signing_bytes()).decode() + if self.signing_payload_base64 != expected: + raise ValueError("TLS enrollment signing payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointPeerTlsEnrollment(_FrozenModel): + statement: CheckpointPeerTlsEnrollmentStatement + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + enrollment_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_enrollment(self) -> CheckpointPeerTlsEnrollment: + _validate_identifier(self.key_id, label="key_id") + _decode_base64(self.signature_base64, label="TLS enrollment signature", length=64) + if self.enrollment_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("TLS enrollment fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"enrollment_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointPeerTlsRevocationStatement(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + peer_id: str = Field(min_length=1, max_length=200) + generation: int = Field(ge=1) + enrollment_sha256: str = Field(pattern=_SHA256_PATTERN) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + reason: str = Field(min_length=1, max_length=500) + revoked_at: str = Field(min_length=1) + statement_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> CheckpointPeerTlsRevocationStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.peer_id, label="peer_id") + _validate_identifier(self.reason, label="reason") + if self.statement_sha256 != _canonical_sha256( + self.model_dump(mode="json", exclude={"statement_sha256"}) + ): + raise ValueError("TLS revocation statement fingerprint does not match") + return self + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class CheckpointPeerTlsRevocation(_FrozenModel): + statement: CheckpointPeerTlsRevocationStatement + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + revocation_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_revocation(self) -> CheckpointPeerTlsRevocation: + _decode_base64(self.signature_base64, label="TLS revocation signature", length=64) + payload = self.model_dump(mode="json", exclude={"revocation_sha256"}) + if self.revocation_sha256 != _canonical_sha256(payload): + raise ValueError("TLS revocation fingerprint does not match") + return self + + +class CheckpointPeerTlsRevocationTemplate(_FrozenModel): + statement: CheckpointPeerTlsRevocationStatement + eligible_key_ids: tuple[str, ...] = Field(min_length=1) + signing_payload_base64: str = Field(min_length=1) + signatures_completed: Literal[False] = False + + @model_validator(mode="after") + def validate_template(self) -> CheckpointPeerTlsRevocationTemplate: + if tuple(sorted(set(self.eligible_key_ids))) != self.eligible_key_ids: + raise ValueError("eligible peer key IDs must be sorted and unique") + expected = base64.b64encode(self.statement.signing_bytes()).decode() + if self.signing_payload_base64 != expected: + raise ValueError("TLS revocation signing payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_checkpoint_peer_tls_revocation_template( + tls_trust: CheckpointPeerTlsTrust, + peer_trust: CheckpointPeerTrust, + *, + peer_id: str, + generation: int, + reason: str, + revoked_at: str, +) -> CheckpointPeerTlsRevocationTemplate: + """Create a private-key-free request to revoke one enrolled generation.""" + enrollment = next( + (item for item in tls_trust.enrollments + if item.statement.peer_id == peer_id and item.statement.generation == generation), + None, + ) + if enrollment is None: + raise ValueError("TLS revocation target is not enrolled") + active_keys = tuple(sorted(key.key_id for key in _active_peer_keys(peer_trust, peer_id))) + if not active_keys: + raise ValueError("TLS revocation peer has no active identity key") + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": tls_trust.registry_id, + "peer_id": peer_id, + "generation": generation, + "enrollment_sha256": enrollment.enrollment_sha256, + "peer_trust_sha256": tls_trust.peer_trust_sha256, + "reason": reason, + "revoked_at": revoked_at, + } + statement = CheckpointPeerTlsRevocationStatement.model_validate( + {**payload, "statement_sha256": _canonical_sha256(payload)} + ) + return CheckpointPeerTlsRevocationTemplate( + statement=statement, + eligible_key_ids=active_keys, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def _enrollment_identity(enrollment: CheckpointPeerTlsEnrollment) -> tuple[str, int]: + return enrollment.statement.peer_id, enrollment.statement.generation + + +class CheckpointPeerTlsTrust(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + enrollments: tuple[CheckpointPeerTlsEnrollment, ...] = Field(min_length=1) + revocations: tuple[CheckpointPeerTlsRevocation, ...] = () + tls_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_trust(self) -> CheckpointPeerTlsTrust: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + if tuple(sorted(self.enrollments, key=_enrollment_identity)) != self.enrollments: + raise ValueError("TLS enrollments must be sorted") + identities = [_enrollment_identity(item) for item in self.enrollments] + if len(identities) != len(set(identities)): + raise ValueError("TLS enrollment generations must be unique") + for index, enrollment in enumerate(self.enrollments): + statement = enrollment.statement + if statement.registry_id != self.registry_id: + raise ValueError("TLS enrollment uses another registry") + if statement.peer_trust_sha256 != self.peer_trust_sha256: + raise ValueError("TLS enrollment uses another peer trust") + predecessor = next( + ( + candidate + for candidate in self.enrollments[:index] + if candidate.statement.peer_id == statement.peer_id + and candidate.statement.generation == statement.generation - 1 + ), + None, + ) + if statement.generation > 1 and ( + predecessor is None + or statement.previous_enrollment_sha256 != predecessor.enrollment_sha256 + ): + raise ValueError("TLS enrollment predecessor is missing or invalid") + targets = { + (item.statement.peer_id, item.statement.generation, item.statement.enrollment_sha256) + for item in self.revocations + } + if len(targets) != len(self.revocations): + raise ValueError("duplicate TLS revocation target") + enrollment_targets = { + (item.statement.peer_id, item.statement.generation, item.enrollment_sha256) + for item in self.enrollments + } + if not targets.issubset(enrollment_targets): + raise ValueError("TLS revocation target is not enrolled") + active = [self.active_enrollment(peer) for peer in self.peer_ids()] + if len({item.statement.certificate_sha256 for item in active}) != len(active): + raise ValueError("active TLS certificate pins must be unique across peers") + if len({item.statement.spki_sha256 for item in active}) != len(active): + raise ValueError("active TLS SPKI pins must be unique across peers") + if self.tls_trust_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("TLS trust fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"tls_trust_sha256"}) + + def peer_ids(self) -> tuple[str, ...]: + return tuple(sorted({item.statement.peer_id for item in self.enrollments})) + + def active_enrollment(self, peer_id: str) -> CheckpointPeerTlsEnrollment: + matches = [item for item in self.enrollments if item.statement.peer_id == peer_id] + if not matches: + raise ValueError(f"peer has no active TLS enrollment: {peer_id}") + revoked = { + (item.statement.peer_id, item.statement.generation, item.statement.enrollment_sha256) + for item in self.revocations + } + for item in reversed(matches): + if ( + item.statement.peer_id, + item.statement.generation, + item.enrollment_sha256, + ) not in revoked: + return item + raise ValueError(f"peer has no active TLS enrollment: {peer_id}") + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _active_peer_keys( + peer_trust: CheckpointPeerTrust, + peer_id: str, +) -> tuple[CheckpointPeerKey, ...]: + return tuple( + key for key in peer_trust.keys if key.peer_id == peer_id and key.status == "active" + ) + + +def _verify_enrollment_signature( + enrollment: CheckpointPeerTlsEnrollment, + peer_trust: CheckpointPeerTrust, +) -> None: + keys = {key.key_id: key for key in _active_peer_keys(peer_trust, enrollment.statement.peer_id)} + key = keys.get(enrollment.key_id) + if key is None: + raise ValueError("TLS enrollment signature key is not active for peer") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64(key.public_key_base64, label="peer public key", length=32) + ).verify( + _decode_base64( + enrollment.signature_base64, + label="TLS enrollment signature", + length=64, + ), + enrollment.statement.signing_bytes(), + ) + except InvalidSignature as exc: + raise ValueError("TLS enrollment signature is invalid") from exc + + +def _verify_revocation_signature( + revocation: CheckpointPeerTlsRevocation, + peer_trust: CheckpointPeerTrust, +) -> None: + keys = {key.key_id: key for key in _active_peer_keys(peer_trust, revocation.statement.peer_id)} + key = keys.get(revocation.key_id) + if key is None: + raise ValueError("TLS revocation signature key is not active for peer") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64(key.public_key_base64, label="peer public key", length=32) + ).verify( + _decode_base64( + revocation.signature_base64, label="TLS revocation signature", length=64 + ), + revocation.statement.signing_bytes(), + ) + except InvalidSignature as exc: + raise ValueError("TLS revocation signature is invalid") from exc + + +def build_signed_checkpoint_peer_tls_revocation( + template: CheckpointPeerTlsRevocationTemplate, + peer_trust: CheckpointPeerTrust, + *, + key_id: str, + signature_base64: str, +) -> CheckpointPeerTlsRevocation: + """Verify a peer signature and finalize a TLS revocation artifact.""" + if key_id not in template.eligible_key_ids: + raise ValueError("TLS revocation signature key is not eligible") + payload = { + "statement": template.statement.model_dump(mode="json"), + "key_id": key_id, + "signature_base64": signature_base64, + } + revocation = CheckpointPeerTlsRevocation.model_validate( + {**payload, "revocation_sha256": _canonical_sha256(payload)} + ) + _verify_revocation_signature(revocation, peer_trust) + return revocation + + +def build_checkpoint_peer_tls_enrollment_template( + certificate: bytes, + peer_trust: CheckpointPeerTrust, + *, + peer_id: str, + generation: int, + predecessor: CheckpointPeerTlsEnrollment | None = None, +) -> CheckpointPeerTlsEnrollmentTemplate: + """Bind a validated TLS leaf certificate to a peer identity signing request.""" + peer_trust = CheckpointPeerTrust.model_validate(peer_trust.model_dump(mode="json")) + active_keys = _active_peer_keys(peer_trust, peer_id) + if not active_keys: + raise ValueError("TLS enrollment peer has no active identity key") + if generation == 1 and predecessor is not None: + raise ValueError("first TLS enrollment cannot declare a predecessor") + if generation > 1 and ( + predecessor is None + or predecessor.statement.peer_id != peer_id + or predecessor.statement.generation != generation - 1 + ): + raise ValueError("TLS enrollment predecessor does not match generation") + metadata = _certificate_metadata(certificate) + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": peer_trust.registry_id, + "peer_id": peer_id, + "generation": generation, + "previous_enrollment_sha256": ( + predecessor.enrollment_sha256 if predecessor is not None else None + ), + "peer_trust_sha256": peer_trust.peer_trust_sha256, + **metadata, + } + statement = CheckpointPeerTlsEnrollmentStatement.model_validate( + {**payload, "statement_sha256": _canonical_sha256(payload)} + ) + return CheckpointPeerTlsEnrollmentTemplate( + statement=statement, + eligible_key_ids=tuple(sorted(key.key_id for key in active_keys)), + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def build_signed_checkpoint_peer_tls_enrollment( + template: CheckpointPeerTlsEnrollmentTemplate, + certificate: bytes, + peer_trust: CheckpointPeerTrust, + *, + key_id: str, + signature_base64: str, + predecessor: CheckpointPeerTlsEnrollment | None = None, +) -> CheckpointPeerTlsEnrollment: + """Verify an identity signature and finalize a TLS enrollment generation.""" + expected = build_checkpoint_peer_tls_enrollment_template( + certificate, + peer_trust, + peer_id=template.statement.peer_id, + generation=template.statement.generation, + predecessor=predecessor, + ) + if expected.statement != template.statement: + raise ValueError("TLS enrollment template does not match certificate or trust") + if key_id not in template.eligible_key_ids: + raise ValueError("TLS enrollment signature key is not eligible") + payload = { + "statement": template.statement.model_dump(mode="json"), + "key_id": key_id, + "signature_base64": signature_base64, + } + enrollment = CheckpointPeerTlsEnrollment.model_validate( + {**payload, "enrollment_sha256": _canonical_sha256(payload)} + ) + _verify_enrollment_signature(enrollment, peer_trust) + return enrollment + + +def build_checkpoint_peer_tls_trust( + peer_trust: CheckpointPeerTrust, + enrollments: tuple[CheckpointPeerTlsEnrollment, ...], + revocations: tuple[CheckpointPeerTlsRevocation, ...] = (), +) -> CheckpointPeerTlsTrust: + """Verify signed enrollment chains and publish deterministic TLS pins.""" + peer_trust = CheckpointPeerTrust.model_validate(peer_trust.model_dump(mode="json")) + normalized = tuple( + sorted( + ( + CheckpointPeerTlsEnrollment.model_validate(item.model_dump(mode="json")) + for item in enrollments + ), + key=_enrollment_identity, + ) + ) + for enrollment in normalized: + _verify_enrollment_signature(enrollment, peer_trust) + normalized_revocations = tuple( + sorted( + ( + CheckpointPeerTlsRevocation.model_validate(item.model_dump(mode="json")) + for item in revocations + ), + key=lambda item: (item.statement.peer_id, item.statement.generation), + ) + ) + for revocation in normalized_revocations: + if revocation.statement.peer_trust_sha256 != peer_trust.peer_trust_sha256: + raise ValueError("TLS revocation uses another peer trust") + _verify_revocation_signature(revocation, peer_trust) + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": peer_trust.registry_id, + "peer_trust_sha256": peer_trust.peer_trust_sha256, + "enrollments": [item.model_dump(mode="json") for item in normalized], + "revocations": [item.model_dump(mode="json") for item in normalized_revocations], + } + return CheckpointPeerTlsTrust.model_validate( + {**payload, "tls_trust_sha256": _canonical_sha256(payload)} + ) + + +def verify_checkpoint_peer_tls_trust( + tls_trust: CheckpointPeerTlsTrust, + peer_trust: CheckpointPeerTrust, +) -> None: + """Reverify every enrollment signature against an exact peer trust artifact.""" + verified = build_checkpoint_peer_tls_trust( + peer_trust, tls_trust.enrollments, tls_trust.revocations + ) + if verified != tls_trust: + raise ValueError("TLS trust does not match checkpoint peer trust") + + +def certificate_fingerprints(certificate: bytes) -> tuple[str, str]: + """Return the normalized DER certificate and SPKI SHA-256 pins.""" + metadata = _certificate_metadata(certificate) + return str(metadata["certificate_sha256"]), str(metadata["spki_sha256"]) + + +def verify_active_tls_certificate( + certificate: bytes, + tls_trust: CheckpointPeerTlsTrust, + peer_id: str, +) -> None: + """Require a leaf certificate to match both active peer pins.""" + certificate_sha256, spki_sha256 = certificate_fingerprints(certificate) + active = tls_trust.active_enrollment(peer_id).statement + if certificate_sha256 != active.certificate_sha256 or spki_sha256 != active.spki_sha256: + raise ValueError("active peer TLS certificate pin does not match") + + +def resolve_active_tls_peer( + certificate: bytes, + tls_trust: CheckpointPeerTlsTrust, +) -> str: + """Resolve an authenticated leaf to exactly one active peer identity.""" + certificate_sha256, spki_sha256 = certificate_fingerprints(certificate) + matches = [ + peer_id + for peer_id in tls_trust.peer_ids() + if ( + tls_trust.active_enrollment(peer_id).statement.certificate_sha256 == certificate_sha256 + and tls_trust.active_enrollment(peer_id).statement.spki_sha256 == spki_sha256 + ) + ] + if len(matches) != 1: + raise ValueError("TLS certificate does not match an active peer enrollment") + return matches[0] + + +def validate_checkpoint_peer_tls_expiry( + tls_trust: CheckpointPeerTlsTrust, + *, + now: datetime | None = None, + warning_window_seconds: int = 0, +) -> tuple[tuple[str, int, str, int], ...]: + """Reject expired active leaves and return deterministic pre-expiry warnings.""" + if warning_window_seconds < 0: + raise ValueError("warning_window_seconds must be non-negative") + current = (now or datetime.now(UTC)).astimezone(UTC) + warnings: list[tuple[str, int, str, int]] = [] + for peer_id in tls_trust.peer_ids(): + statement = tls_trust.active_enrollment(peer_id).statement + expires = datetime.fromisoformat( + statement.not_valid_after.replace("Z", "+00:00") + ).astimezone(UTC) + seconds = int((expires - current).total_seconds()) + if seconds < 0: + raise ValueError(f"active TLS certificate is expired: {peer_id}") + if seconds <= warning_window_seconds: + warnings.append((peer_id, statement.generation, statement.not_valid_after, seconds)) + return tuple(warnings) diff --git a/benchmarks/agent_cli_gossip_tls_transport.py b/benchmarks/agent_cli_gossip_tls_transport.py new file mode 100644 index 0000000..b5b7a9d --- /dev/null +++ b/benchmarks/agent_cli_gossip_tls_transport.py @@ -0,0 +1,687 @@ +"""TLS 1.3 mutual-auth transport for signed checkpoint gossip.""" + +from __future__ import annotations + +import asyncio +import base64 +import ipaddress +import json +import os +import secrets +import ssl +import stat +from contextlib import suppress +from datetime import datetime +from pathlib import Path +from typing import Protocol, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsTrust, + certificate_fingerprints, + resolve_active_tls_peer, + validate_checkpoint_peer_tls_expiry, + verify_active_tls_certificate, +) + +GOSSIP_MTLS_PROTOCOL_VERSION = 2 +MAX_GOSSIP_REQUEST_BYTES = 64 * 1024 +MAX_GOSSIP_RESPONSE_BYTES = 2 * 1024 * 1024 +MAX_GOSSIP_RETAINED_NONCES = 1024 +MAX_GOSSIP_CONCURRENT_CLIENTS = 8 +DEFAULT_GOSSIP_REQUEST_TIMEOUT_SECONDS = 2.0 + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _validate_identifier(value: str, *, label: str) -> None: + if not value or value != value.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +def _validated_addresses(addresses: frozenset[str], *, label: str) -> frozenset[str]: + if not addresses: + raise ValueError(f"checkpoint gossip {label} allowlist must not be empty") + normalized: set[str] = set() + for address in addresses: + try: + normalized.add(str(ipaddress.ip_address(address))) + except ValueError as exc: + raise ValueError(f"checkpoint gossip {label} allowlist is invalid") from exc + if normalized != set(addresses): + raise ValueError(f"checkpoint gossip {label} allowlist must be canonical") + return frozenset(normalized) + + +def _read_certificate(path: Path) -> bytes: + try: + return path.read_bytes() + except OSError as exc: + raise ValueError("TLS certificate cannot be read") from exc + + +def _require_private_key_permissions(path: Path) -> None: + try: + metadata = path.lstat() + except OSError as exc: + raise ValueError("TLS private key cannot be read") from exc + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise ValueError("TLS private key must be a regular non-symlink file") + mode = stat.S_IMODE(metadata.st_mode) + if mode & 0o077: + raise ValueError("TLS private key permissions must not allow group or other access") + + +class CheckpointMutualTlsGossipDescriptor(_FrozenModel): + protocol_version: int + transport: str + host: str + port: int = Field(ge=1, le=65535) + registry_id: str = Field(min_length=1, max_length=200) + server_peer_id: str = Field(min_length=1, max_length=200) + server_certificate_sha256: str = Field(pattern=_SHA256_PATTERN) + server_spki_sha256: str = Field(pattern=_SHA256_PATTERN) + tls_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + instance_id: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_descriptor(self) -> CheckpointMutualTlsGossipDescriptor: + if self.protocol_version != GOSSIP_MTLS_PROTOCOL_VERSION: + raise ValueError("unsupported checkpoint mTLS gossip protocol") + if self.transport != "mtls": + raise ValueError("checkpoint gossip transport must be mTLS") + try: + if str(ipaddress.ip_address(self.host)) != self.host: + raise ValueError + except ValueError as exc: + raise ValueError("checkpoint gossip descriptor host must be a canonical IP") from exc + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.server_peer_id, label="server_peer_id") + return self + + +class _MutualTlsRequest(_FrozenModel): + protocol_version: int + registry_id: str + server_peer_id: str + client_peer_id: str + instance_id: str + client_nonce: str + operation: str = Field(min_length=1, max_length=100) + payload: dict[str, object] + + @model_validator(mode="after") + def validate_request(self) -> _MutualTlsRequest: + if self.protocol_version != GOSSIP_MTLS_PROTOCOL_VERSION: + raise ValueError("unsupported checkpoint mTLS gossip protocol") + for value, label in ( + (self.registry_id, "registry_id"), + (self.server_peer_id, "server_peer_id"), + (self.client_peer_id, "client_peer_id"), + (self.operation, "operation"), + ): + _validate_identifier(value, label=label) + try: + nonce = base64.b64decode(self.client_nonce, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError("checkpoint gossip client nonce is invalid") from exc + if len(nonce) != 32 or base64.b64encode(nonce).decode() != self.client_nonce: + raise ValueError("checkpoint gossip client nonce must encode 32 bytes") + return self + + +class CheckpointMutualTlsGossipRequestHandler(Protocol): + async def dispatch( + self, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: ... + + +class CheckpointMutualTlsGossipServer: + """Serve bounded requests after CA, hostname, peer pin, and IP authentication.""" + + def __init__( + self, + *, + descriptor_path: Path, + bind_host: str, + advertised_host: str, + registry_id: str, + server_peer_id: str, + handler: CheckpointMutualTlsGossipRequestHandler, + tls_trust: CheckpointPeerTlsTrust, + certificate_path: Path, + private_key_path: Path, + certificate_authority_path: Path, + allowed_client_addresses: frozenset[str], + request_timeout_seconds: float = DEFAULT_GOSSIP_REQUEST_TIMEOUT_SECONDS, + max_requests: int = 64, + max_concurrent_clients: int = MAX_GOSSIP_CONCURRENT_CLIENTS, + now: datetime | None = None, + expiry_warning_window_seconds: int = 0, + ) -> None: + _validate_identifier(registry_id, label="registry_id") + _validate_identifier(server_peer_id, label="server_peer_id") + if tls_trust.registry_id != registry_id: + raise ValueError("TLS trust uses another registry") + self.expiry_warnings = validate_checkpoint_peer_tls_expiry( + tls_trust, now=now, warning_window_seconds=expiry_warning_window_seconds + ) + try: + bind_host = str(ipaddress.ip_address(bind_host)) + advertised_host = str(ipaddress.ip_address(advertised_host)) + except ValueError as exc: + raise ValueError("checkpoint gossip bind and advertised hosts must be IPs") from exc + if not 0 < request_timeout_seconds <= 30: + raise ValueError("checkpoint gossip timeout must be in (0, 30] seconds") + if not 1 <= max_requests <= MAX_GOSSIP_RETAINED_NONCES: + raise ValueError("checkpoint gossip max_requests is out of bounds") + if not 1 <= max_concurrent_clients <= 64: + raise ValueError("checkpoint gossip max concurrent clients is out of bounds") + _require_private_key_permissions(private_key_path) + certificate = _read_certificate(certificate_path) + verify_active_tls_certificate(certificate, tls_trust, server_peer_id) + certificate_sha256, spki_sha256 = certificate_fingerprints(certificate) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_3 + context.maximum_version = ssl.TLSVersion.TLSv1_3 + context.verify_mode = ssl.CERT_REQUIRED + try: + context.load_cert_chain(certificate_path, private_key_path) + context.load_verify_locations(cafile=certificate_authority_path) + except (OSError, ssl.SSLError) as exc: + raise ValueError("TLS server credentials are invalid") from exc + + self.descriptor_path = descriptor_path + self.bind_host = bind_host + self.advertised_host = advertised_host + self.registry_id = registry_id + self.server_peer_id = server_peer_id + self._handler = handler + self._tls_trust = tls_trust + self._certificate_sha256 = certificate_sha256 + self._spki_sha256 = spki_sha256 + self._allowed_client_addresses = _validated_addresses( + allowed_client_addresses, + label="client address", + ) + self._request_timeout_seconds = request_timeout_seconds + self._max_requests = max_requests + self._max_concurrent_clients = max_concurrent_clients + self._ssl_context = context + self._instance_id = secrets.token_hex(32) + self._owned_descriptor_text: str | None = None + self._server: asyncio.Server | None = None + self._client_tasks: set[asyncio.Task[object]] = set() + self._client_writers: set[asyncio.StreamWriter] = set() + self._used_client_nonces: set[str] = set() + self._counter_lock = asyncio.Lock() + self._stopped = asyncio.Event() + self._completed_requests = 0 + + @property + def completed_requests(self) -> int: + return self._completed_requests + + @property + def active_client_count(self) -> int: + return len(self._client_writers) + + async def __aenter__(self) -> Self: + await self.start() + return self + + async def __aexit__(self, *_exc_info: object) -> None: + await self.close() + + async def start(self) -> None: + if self._server is not None: + raise RuntimeError("checkpoint mTLS gossip server is already running") + if self._stopped.is_set(): + raise RuntimeError("checkpoint mTLS gossip server cannot be restarted") + server = await asyncio.start_server( + self._handle_client, + host=self.bind_host, + port=0, + ssl=self._ssl_context, + ssl_handshake_timeout=0.25, + limit=MAX_GOSSIP_REQUEST_BYTES, + ) + sockets = server.sockets + if not sockets: + server.close() + await server.wait_closed() + raise RuntimeError("checkpoint mTLS gossip server did not bind a socket") + self._server = server + try: + self._write_descriptor(int(sockets[0].getsockname()[1])) + except Exception: + await self.close() + raise + + async def wait_stopped(self) -> None: + await self._stopped.wait() + + async def serve_until_stopped( + self, + *, + lifetime_timeout_seconds: float | None = None, + ) -> None: + if self._server is None: + raise RuntimeError("checkpoint mTLS gossip server is not running") + try: + if lifetime_timeout_seconds is None: + await self.wait_stopped() + else: + if not 0 < lifetime_timeout_seconds <= 3600: + raise ValueError("checkpoint gossip lifetime must be in (0, 3600] seconds") + with suppress(TimeoutError): + await asyncio.wait_for(self.wait_stopped(), timeout=lifetime_timeout_seconds) + finally: + await self.close() + + async def close(self) -> None: + server = self._server + self._server = None + if server is not None: + server.close() + self._stopped.set() + current = asyncio.current_task() + tasks = tuple(task for task in self._client_tasks if task is not current) + for writer in tuple(self._client_writers): + writer.close() + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + if server is not None: + await server.wait_closed() + for writer in tuple(self._client_writers): + with suppress(ConnectionError): + await writer.wait_closed() + self._remove_owned_descriptor() + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + task = asyncio.current_task() + if task is not None: + self._client_tasks.add(task) + self._client_writers.add(writer) + should_stop = False + client_nonce: str | None = None + try: + peername = writer.get_extra_info("peername") + peer_address = str(ipaddress.ip_address(peername[0])) if peername else "" + if peer_address not in self._allowed_client_addresses: + await self._write_response(writer, {"error": "address_not_allowed", "ok": False}) + return + if len(self._client_writers) > self._max_concurrent_clients: + await self._write_response(writer, {"error": "server_busy", "ok": False}) + return + ssl_object = writer.get_extra_info("ssl_object") + if ssl_object is None or ssl_object.version() != "TLSv1.3": + raise ValueError("TLS 1.3 session is required") + peer_certificate = ssl_object.getpeercert(binary_form=True) + if not peer_certificate: + raise ValueError("client TLS certificate is missing") + authenticated_peer_id = resolve_active_tls_peer(peer_certificate, self._tls_trust) + raw = await self._read_request_line(reader) + request = _MutualTlsRequest.model_validate_json(raw) + client_nonce = request.client_nonce + if ( + request.registry_id != self.registry_id + or request.server_peer_id != self.server_peer_id + or request.instance_id != self._instance_id + or request.client_peer_id != authenticated_peer_id + ): + await self._write_response( + writer, + self._response(client_nonce, error="endpoint_or_peer_mismatch"), + ) + return + async with self._counter_lock: + if client_nonce in self._used_client_nonces: + await self._write_response( + writer, self._response(client_nonce, error="replayed_nonce") + ) + return + if len(self._used_client_nonces) >= MAX_GOSSIP_RETAINED_NONCES: + await self._write_response( + writer, + self._response(client_nonce, error="nonce_capacity_exhausted"), + ) + return + if self._completed_requests >= self._max_requests: + await self._write_response( + writer, + self._response(client_nonce, error="request_capacity_exhausted"), + ) + return + self._used_client_nonces.add(client_nonce) + self._completed_requests += 1 + should_stop = self._completed_requests >= self._max_requests + try: + result = await asyncio.wait_for( + self._handler.dispatch(request.operation, request.payload), + timeout=self._request_timeout_seconds, + ) + response = self._response(client_nonce, result=result) + if len(_canonical_json(response).encode()) > MAX_GOSSIP_RESPONSE_BYTES - 1: + response = self._response(client_nonce, error="response_too_large") + await self._write_response(writer, response) + except TimeoutError: + await self._write_response( + writer, self._response(client_nonce, error="request_timeout") + ) + except Exception: + await self._write_response( + writer, self._response(client_nonce, error="internal_error") + ) + except TimeoutError: + with suppress(ConnectionError): + await self._write_response(writer, {"error": "request_timeout", "ok": False}) + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + with suppress(ConnectionError): + await self._write_response(writer, {"error": "invalid_request", "ok": False}) + except ConnectionError: + pass + except asyncio.CancelledError: + raise + finally: + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + self._client_writers.discard(writer) + if task is not None: + self._client_tasks.discard(task) + if should_stop: + server = self._server + if server is not None: + server.close() + self._stopped.set() + + async def _read_request_line(self, reader: asyncio.StreamReader) -> bytes: + raw = await asyncio.wait_for(reader.readline(), timeout=self._request_timeout_seconds) + if not raw or len(raw) > MAX_GOSSIP_REQUEST_BYTES or not raw.endswith(b"\n"): + raise ValueError("checkpoint gossip request size is invalid") + return raw + + def _response( + self, + client_nonce: str, + *, + result: dict[str, object] | None = None, + error: str | None = None, + ) -> dict[str, object]: + response: dict[str, object] = { + "client_nonce": client_nonce, + "instance_id": self._instance_id, + "ok": error is None, + "protocol_version": GOSSIP_MTLS_PROTOCOL_VERSION, + "registry_id": self.registry_id, + "server_peer_id": self.server_peer_id, + } + if error is None: + response["result"] = result + else: + response["error"] = error + return response + + @staticmethod + async def _write_response( + writer: asyncio.StreamWriter, + response: dict[str, object], + ) -> None: + encoded = _canonical_json(response).encode() + b"\n" + if len(encoded) > MAX_GOSSIP_RESPONSE_BYTES: + raise ValueError("checkpoint gossip response exceeds protocol limit") + writer.write(encoded) + await writer.drain() + + def _write_descriptor(self, port: int) -> None: + path = self.descriptor_path + path.parent.mkdir(parents=True, exist_ok=True) + path.parent.chmod(0o700) + if path.exists(): + raise ValueError("checkpoint gossip descriptor already exists") + descriptor = CheckpointMutualTlsGossipDescriptor( + protocol_version=GOSSIP_MTLS_PROTOCOL_VERSION, + transport="mtls", + host=self.advertised_host, + port=port, + registry_id=self.registry_id, + server_peer_id=self.server_peer_id, + server_certificate_sha256=self._certificate_sha256, + server_spki_sha256=self._spki_sha256, + tls_trust_sha256=self._tls_trust.tls_trust_sha256, + instance_id=self._instance_id, + ) + descriptor_text = _canonical_json(descriptor.model_dump(mode="json")) + "\n" + temporary = path.with_suffix(f".{self._instance_id[:12]}.tmp") + try: + temporary.write_text(descriptor_text, encoding="utf-8") + temporary.chmod(0o600) + try: + os.link(temporary, path) + except FileExistsError as exc: + raise ValueError("checkpoint gossip descriptor already exists") from exc + finally: + temporary.unlink(missing_ok=True) + path.chmod(0o600) + self._owned_descriptor_text = descriptor_text + + def _remove_owned_descriptor(self) -> None: + if self._owned_descriptor_text is None: + return + try: + descriptor_text = self.descriptor_path.read_text(encoding="utf-8") + except OSError: + return + if descriptor_text == self._owned_descriptor_text: + self.descriptor_path.unlink(missing_ok=True) + + +class CheckpointMutualTlsGossipClient: + """Reusable request sender with one pinned mutual-TLS configuration.""" + + def __init__( + self, + *, + descriptor_path: Path, + client_peer_id: str, + tls_trust: CheckpointPeerTlsTrust, + certificate_path: Path, + private_key_path: Path, + certificate_authority_path: Path, + server_hostname: str, + allowed_server_addresses: frozenset[str], + request_timeout_seconds: float = DEFAULT_GOSSIP_REQUEST_TIMEOUT_SECONDS, + now: datetime | None = None, + expiry_warning_window_seconds: int = 0, + ) -> None: + _validate_identifier(client_peer_id, label="client_peer_id") + _validate_identifier(server_hostname, label="server_hostname") + if not 0 < request_timeout_seconds <= 30: + raise ValueError("checkpoint gossip timeout must be in (0, 30] seconds") + self.descriptor_path = descriptor_path + self.client_peer_id = client_peer_id + self.tls_trust = CheckpointPeerTlsTrust.model_validate(tls_trust.model_dump(mode="json")) + self.expiry_warnings = validate_checkpoint_peer_tls_expiry( + self.tls_trust, now=now, warning_window_seconds=expiry_warning_window_seconds + ) + self.certificate_path = certificate_path + self.private_key_path = private_key_path + self.certificate_authority_path = certificate_authority_path + self.server_hostname = server_hostname + self.allowed_server_addresses = _validated_addresses( + allowed_server_addresses, + label="server address", + ) + self.request_timeout_seconds = request_timeout_seconds + + async def __call__( + self, + *, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: + return await send_checkpoint_mtls_gossip_request( + descriptor_path=self.descriptor_path, + client_peer_id=self.client_peer_id, + tls_trust=self.tls_trust, + certificate_path=self.certificate_path, + private_key_path=self.private_key_path, + certificate_authority_path=self.certificate_authority_path, + server_hostname=self.server_hostname, + allowed_server_addresses=self.allowed_server_addresses, + operation=operation, + payload=payload, + request_timeout_seconds=self.request_timeout_seconds, + ) + + +async def send_checkpoint_mtls_gossip_request( + *, + descriptor_path: Path, + client_peer_id: str, + tls_trust: CheckpointPeerTlsTrust, + certificate_path: Path, + private_key_path: Path, + certificate_authority_path: Path, + server_hostname: str, + allowed_server_addresses: frozenset[str], + operation: str, + payload: dict[str, object], + client_nonce: bytes | None = None, + request_timeout_seconds: float = DEFAULT_GOSSIP_REQUEST_TIMEOUT_SECONDS, +) -> dict[str, object]: + """Send one request only after mutual CA, hostname, address, and peer-pin checks.""" + if not 0 < request_timeout_seconds <= 30: + raise ValueError("checkpoint gossip timeout must be in (0, 30] seconds") + _validate_identifier(client_peer_id, label="client_peer_id") + _validate_identifier(operation, label="operation") + _validate_identifier(server_hostname, label="server_hostname") + addresses = _validated_addresses(allowed_server_addresses, label="server address") + if stat.S_IMODE(descriptor_path.stat().st_mode) != 0o600: + raise ValueError("checkpoint gossip descriptor permissions must be 0600") + descriptor = CheckpointMutualTlsGossipDescriptor.model_validate_json( + descriptor_path.read_text(encoding="utf-8") + ) + if descriptor.host not in addresses: + raise ValueError("checkpoint gossip endpoint is outside the server address allowlist") + if ( + descriptor.registry_id != tls_trust.registry_id + or descriptor.tls_trust_sha256 != tls_trust.tls_trust_sha256 + ): + raise ValueError("checkpoint gossip descriptor does not match TLS trust") + active_server = tls_trust.active_enrollment(descriptor.server_peer_id).statement + if ( + descriptor.server_certificate_sha256 != active_server.certificate_sha256 + or descriptor.server_spki_sha256 != active_server.spki_sha256 + ): + raise ValueError("checkpoint gossip descriptor active certificate pin is invalid") + _require_private_key_permissions(private_key_path) + verify_active_tls_certificate(_read_certificate(certificate_path), tls_trust, client_peer_id) + nonce = client_nonce if client_nonce is not None else secrets.token_bytes(32) + if len(nonce) != 32: + raise ValueError("checkpoint gossip client nonce must contain 32 bytes") + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.minimum_version = ssl.TLSVersion.TLSv1_3 + context.maximum_version = ssl.TLSVersion.TLSv1_3 + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + try: + context.load_verify_locations(cafile=certificate_authority_path) + context.load_cert_chain(certificate_path, private_key_path) + except (OSError, ssl.SSLError) as exc: + raise ValueError("TLS client credentials are invalid") from exc + + writer: asyncio.StreamWriter | None = None + try: + reader, connected_writer = await asyncio.wait_for( + asyncio.open_connection( + descriptor.host, + descriptor.port, + ssl=context, + server_hostname=server_hostname, + ssl_handshake_timeout=request_timeout_seconds, + limit=MAX_GOSSIP_RESPONSE_BYTES, + ), + timeout=request_timeout_seconds, + ) + writer = connected_writer + peername = connected_writer.get_extra_info("peername") + peer_address = str(ipaddress.ip_address(peername[0])) if peername else "" + if peer_address not in addresses: + raise ValueError("checkpoint gossip connected endpoint is outside allowlist") + ssl_object = connected_writer.get_extra_info("ssl_object") + if ssl_object is None or ssl_object.version() != "TLSv1.3": + raise RuntimeError("checkpoint gossip TLS 1.3 negotiation failed") + server_certificate = ssl_object.getpeercert(binary_form=True) + if not server_certificate: + raise RuntimeError("checkpoint gossip server certificate is missing") + verify_active_tls_certificate(server_certificate, tls_trust, descriptor.server_peer_id) + request = _MutualTlsRequest( + protocol_version=GOSSIP_MTLS_PROTOCOL_VERSION, + registry_id=descriptor.registry_id, + server_peer_id=descriptor.server_peer_id, + client_peer_id=client_peer_id, + instance_id=descriptor.instance_id, + client_nonce=base64.b64encode(nonce).decode(), + operation=operation, + payload=payload, + ) + encoded = _canonical_json(request.model_dump(mode="json")).encode() + b"\n" + if len(encoded) > MAX_GOSSIP_REQUEST_BYTES: + raise ValueError("checkpoint gossip request exceeds protocol limit") + connected_writer.write(encoded) + await asyncio.wait_for(connected_writer.drain(), timeout=request_timeout_seconds) + raw = await asyncio.wait_for(reader.readline(), timeout=request_timeout_seconds) + if not raw or len(raw) > MAX_GOSSIP_RESPONSE_BYTES or not raw.endswith(b"\n"): + raise RuntimeError("checkpoint gossip endpoint returned an invalid response") + response = json.loads(raw) + if not isinstance(response, dict): + raise RuntimeError("checkpoint gossip endpoint returned an invalid response") + expected = { + "protocol_version": GOSSIP_MTLS_PROTOCOL_VERSION, + "registry_id": descriptor.registry_id, + "server_peer_id": descriptor.server_peer_id, + "instance_id": descriptor.instance_id, + "client_nonce": request.client_nonce, + } + if any(response.get(key) != value for key, value in expected.items()): + raise RuntimeError("checkpoint gossip endpoint response does not match request") + if response.get("ok") is not True: + raise RuntimeError( + f"checkpoint gossip request rejected: {response.get('error', 'unknown_error')}" + ) + result = response.get("result") + if not isinstance(result, dict): + raise RuntimeError("checkpoint gossip endpoint returned an invalid result") + return result + except ssl.SSLError as exc: + raise RuntimeError("checkpoint gossip TLS certificate or endpoint failed") from exc + except ValueError: + raise + except (OSError, TimeoutError) as exc: + raise RuntimeError("checkpoint gossip TLS certificate or endpoint failed") from exc + finally: + if writer is not None: + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() diff --git a/benchmarks/agent_cli_gossip_transport.py b/benchmarks/agent_cli_gossip_transport.py new file mode 100644 index 0000000..3893aad --- /dev/null +++ b/benchmarks/agent_cli_gossip_transport.py @@ -0,0 +1,729 @@ +"""Bounded authenticated loopback transport for signed checkpoint gossip.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import hmac +import json +import secrets +import stat +from contextlib import suppress +from pathlib import Path +from typing import Any, Protocol, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +GOSSIP_PROTOCOL_VERSION = 1 +GOSSIP_LOOPBACK_HOST = "127.0.0.1" +MAX_GOSSIP_REQUEST_BYTES = 64 * 1024 +MAX_GOSSIP_RESPONSE_BYTES = 2 * 1024 * 1024 +MAX_GOSSIP_RETAINED_NONCES = 1024 +MAX_GOSSIP_CONCURRENT_CLIENTS = 8 +DEFAULT_GOSSIP_REQUEST_TIMEOUT_SECONDS = 2.0 + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if not identifier or identifier != identifier.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +def _authentication_sha256(token: bytes, payload: object) -> str: + return hmac.new( + token, + _canonical_json(payload).encode("utf-8"), + hashlib.sha256, + ).hexdigest() + + +class CheckpointGossipDescriptor(_FrozenModel): + protocol_version: int + host: str + port: int = Field(ge=1, le=65535) + registry_id: str = Field(min_length=1, max_length=200) + source_peer_id: str = Field(min_length=1, max_length=200) + instance_id: str = Field(pattern=_SHA256_PATTERN) + token: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_descriptor(self) -> CheckpointGossipDescriptor: + if self.protocol_version != GOSSIP_PROTOCOL_VERSION: + raise ValueError("unsupported checkpoint gossip protocol") + if self.host != GOSSIP_LOOPBACK_HOST: + raise ValueError("checkpoint gossip host must be loopback") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.source_peer_id, label="source_peer_id") + _decode_base64(self.token, label="checkpoint gossip token", length=32) + return self + + +class _ChallengeRequest(_FrozenModel): + phase: str + protocol_version: int + registry_id: str + source_peer_id: str + instance_id: str + client_nonce: str + + @model_validator(mode="after") + def validate_challenge(self) -> _ChallengeRequest: + if self.phase != "challenge": + raise ValueError("checkpoint gossip challenge phase is invalid") + if self.protocol_version != GOSSIP_PROTOCOL_VERSION: + raise ValueError("unsupported checkpoint gossip protocol") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.source_peer_id, label="source_peer_id") + if not self.instance_id: + raise ValueError("checkpoint gossip instance_id is required") + _decode_base64(self.client_nonce, label="checkpoint gossip client nonce", length=32) + return self + + +class _AuthenticatedRequest(_FrozenModel): + phase: str + protocol_version: int + registry_id: str + source_peer_id: str + instance_id: str + client_nonce: str + server_nonce: str + operation: str = Field(min_length=1, max_length=100) + payload: dict[str, object] + authentication_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_request(self) -> _AuthenticatedRequest: + if self.phase != "request": + raise ValueError("checkpoint gossip request phase is invalid") + if self.protocol_version != GOSSIP_PROTOCOL_VERSION: + raise ValueError("unsupported checkpoint gossip protocol") + _validate_identifier(self.registry_id, label="registry_id") + _validate_identifier(self.source_peer_id, label="source_peer_id") + _validate_identifier(self.operation, label="operation") + _decode_base64(self.client_nonce, label="checkpoint gossip client nonce", length=32) + _decode_base64(self.server_nonce, label="checkpoint gossip server nonce", length=32) + return self + + def binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"authentication_sha256"}) + + +class CheckpointGossipRequestHandler(Protocol): + async def dispatch( + self, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: ... + + +class CheckpointGossipServer: + """Serve bounded challenge-authenticated requests on one loopback socket.""" + + def __init__( + self, + *, + descriptor_path: Path, + registry_id: str, + source_peer_id: str, + handler: CheckpointGossipRequestHandler, + request_timeout_seconds: float = DEFAULT_GOSSIP_REQUEST_TIMEOUT_SECONDS, + max_requests: int = 64, + max_concurrent_clients: int = MAX_GOSSIP_CONCURRENT_CLIENTS, + ) -> None: + _validate_identifier(registry_id, label="registry_id") + _validate_identifier(source_peer_id, label="source_peer_id") + if not 0 < request_timeout_seconds <= 30: + raise ValueError("checkpoint gossip timeout must be in (0, 30] seconds") + if not 1 <= max_requests <= MAX_GOSSIP_RETAINED_NONCES: + raise ValueError("checkpoint gossip max_requests is out of bounds") + if not 1 <= max_concurrent_clients <= 64: + raise ValueError("checkpoint gossip max concurrent clients is out of bounds") + self.descriptor_path = descriptor_path + self.registry_id = registry_id + self.source_peer_id = source_peer_id + self._handler = handler + self._request_timeout_seconds = request_timeout_seconds + self._max_requests = max_requests + self._max_concurrent_clients = max_concurrent_clients + self._token = secrets.token_bytes(32) + self._instance_id = secrets.token_hex(32) + self._server: asyncio.Server | None = None + self._client_tasks: set[asyncio.Task[object]] = set() + self._client_writers: set[asyncio.StreamWriter] = set() + self._used_client_nonces: set[str] = set() + self._counter_lock = asyncio.Lock() + self._stopped = asyncio.Event() + self._completed_requests = 0 + + @property + def completed_requests(self) -> int: + return self._completed_requests + + @property + def active_client_count(self) -> int: + return len(self._client_writers) + + async def __aenter__(self) -> Self: + await self.start() + return self + + async def __aexit__(self, *_exc_info: object) -> None: + await self.close() + + async def start(self) -> None: + if self._server is not None: + raise RuntimeError("checkpoint gossip server is already running") + if self._stopped.is_set(): + raise RuntimeError("checkpoint gossip server cannot be restarted") + server = await asyncio.start_server( + self._handle_client, + host=GOSSIP_LOOPBACK_HOST, + port=0, + limit=MAX_GOSSIP_REQUEST_BYTES, + ) + sockets = server.sockets + if not sockets: + server.close() + await server.wait_closed() + raise RuntimeError("checkpoint gossip server did not bind a socket") + self._server = server + try: + self._write_descriptor(int(sockets[0].getsockname()[1])) + except Exception: + await self.close() + raise + + async def wait_stopped(self) -> None: + await self._stopped.wait() + + async def serve_until_stopped( + self, + *, + lifetime_timeout_seconds: float | None = None, + ) -> None: + if self._server is None: + raise RuntimeError("checkpoint gossip server is not running") + try: + if lifetime_timeout_seconds is None: + await self.wait_stopped() + else: + if not 0 < lifetime_timeout_seconds <= 3600: + raise ValueError( + "checkpoint gossip lifetime must be in (0, 3600] seconds" + ) + with suppress(TimeoutError): + await asyncio.wait_for( + self.wait_stopped(), + timeout=lifetime_timeout_seconds, + ) + finally: + await self.close() + + async def close(self) -> None: + server = self._server + self._server = None + if server is not None: + server.close() + self._stopped.set() + current = asyncio.current_task() + tasks = tuple(task for task in self._client_tasks if task is not current) + for writer in tuple(self._client_writers): + writer.close() + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + if server is not None: + await server.wait_closed() + for writer in tuple(self._client_writers): + with suppress(ConnectionError): + await writer.wait_closed() + self._remove_owned_descriptor() + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + task = asyncio.current_task() + if task is not None: + self._client_tasks.add(task) + self._client_writers.add(writer) + should_stop = False + try: + if len(self._client_writers) > self._max_concurrent_clients: + await self._write_response( + writer, + {"error": "server_busy", "ok": False}, + ) + return + challenge_raw = await self._read_request_line(reader) + challenge = _ChallengeRequest.model_validate_json(challenge_raw) + if ( + challenge.registry_id != self.registry_id + or challenge.source_peer_id != self.source_peer_id + or challenge.instance_id != self._instance_id + ): + await self._write_response( + writer, + {"error": "endpoint_mismatch", "ok": False}, + ) + return + if challenge.client_nonce in self._used_client_nonces: + await self._write_response( + writer, + {"error": "replayed_nonce", "ok": False}, + ) + return + if len(self._used_client_nonces) >= MAX_GOSSIP_RETAINED_NONCES: + await self._write_response( + writer, + {"error": "nonce_capacity_exhausted", "ok": False}, + ) + return + self._used_client_nonces.add(challenge.client_nonce) + server_nonce = base64.b64encode(secrets.token_bytes(32)).decode() + challenge_response = { + "client_nonce": challenge.client_nonce, + "instance_id": self._instance_id, + "ok": True, + "phase": "challenge", + "protocol_version": GOSSIP_PROTOCOL_VERSION, + "registry_id": self.registry_id, + "server_nonce": server_nonce, + "source_peer_id": self.source_peer_id, + } + await self._write_authenticated_response(writer, challenge_response) + + request_raw = await self._read_request_line(reader) + request = _AuthenticatedRequest.model_validate_json(request_raw) + if ( + request.registry_id != self.registry_id + or request.source_peer_id != self.source_peer_id + or request.instance_id != self._instance_id + or request.client_nonce != challenge.client_nonce + or request.server_nonce != server_nonce + ): + await self._write_authenticated_error( + writer, + challenge.client_nonce, + server_nonce, + "challenge_mismatch", + ) + return + expected_authentication = _authentication_sha256( + self._token, + request.binding_payload(), + ) + if not hmac.compare_digest( + request.authentication_sha256, + expected_authentication, + ): + await self._write_authenticated_error( + writer, + challenge.client_nonce, + server_nonce, + "unauthorized", + ) + return + async with self._counter_lock: + if self._completed_requests >= self._max_requests: + await self._write_authenticated_error( + writer, + challenge.client_nonce, + server_nonce, + "request_capacity_exhausted", + ) + return + self._completed_requests += 1 + should_stop = self._completed_requests >= self._max_requests + try: + result = await asyncio.wait_for( + self._handler.dispatch(request.operation, request.payload), + timeout=self._request_timeout_seconds, + ) + response = self._response_payload( + client_nonce=challenge.client_nonce, + server_nonce=server_nonce, + result=result, + ) + if len(_canonical_json(response).encode("utf-8")) > ( + MAX_GOSSIP_RESPONSE_BYTES - 128 + ): + await self._write_authenticated_error( + writer, + challenge.client_nonce, + server_nonce, + "response_too_large", + ) + else: + await self._write_authenticated_response(writer, response) + except TimeoutError: + await self._write_authenticated_error( + writer, + challenge.client_nonce, + server_nonce, + "request_timeout", + ) + except GossipRequestError as exc: + await self._write_authenticated_error( + writer, + challenge.client_nonce, + server_nonce, + exc.code, + ) + except Exception: + await self._write_authenticated_error( + writer, + challenge.client_nonce, + server_nonce, + "internal_error", + ) + except TimeoutError: + with suppress(ConnectionError): + await self._write_response( + writer, + {"error": "request_timeout", "ok": False}, + ) + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + with suppress(ConnectionError): + await self._write_response( + writer, + {"error": "invalid_request", "ok": False}, + ) + except ConnectionError: + pass + except asyncio.CancelledError: + raise + finally: + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + self._client_writers.discard(writer) + if task is not None: + self._client_tasks.discard(task) + if should_stop: + server = self._server + if server is not None: + server.close() + self._stopped.set() + + async def _read_request_line(self, reader: asyncio.StreamReader) -> bytes: + raw = await asyncio.wait_for( + reader.readline(), + timeout=self._request_timeout_seconds, + ) + if not raw or len(raw) > MAX_GOSSIP_REQUEST_BYTES or not raw.endswith(b"\n"): + raise ValueError("checkpoint gossip request size is invalid") + return raw + + async def _write_authenticated_error( + self, + writer: asyncio.StreamWriter, + client_nonce: str, + server_nonce: str, + error: str, + ) -> None: + await self._write_authenticated_response( + writer, + { + "client_nonce": client_nonce, + "error": error, + "instance_id": self._instance_id, + "ok": False, + "phase": "response", + "protocol_version": GOSSIP_PROTOCOL_VERSION, + "registry_id": self.registry_id, + "server_nonce": server_nonce, + "source_peer_id": self.source_peer_id, + }, + ) + + def _response_payload( + self, + *, + client_nonce: str, + server_nonce: str, + result: dict[str, object], + ) -> dict[str, object]: + return { + "client_nonce": client_nonce, + "instance_id": self._instance_id, + "ok": True, + "phase": "response", + "protocol_version": GOSSIP_PROTOCOL_VERSION, + "registry_id": self.registry_id, + "result": result, + "server_nonce": server_nonce, + "source_peer_id": self.source_peer_id, + } + + async def _write_authenticated_response( + self, + writer: asyncio.StreamWriter, + response: dict[str, object], + ) -> None: + authenticated = { + **response, + "authentication_sha256": _authentication_sha256(self._token, response), + } + await self._write_response(writer, authenticated) + + @staticmethod + async def _write_response( + writer: asyncio.StreamWriter, + response: dict[str, object], + ) -> None: + encoded = _canonical_json(response).encode("utf-8") + b"\n" + if len(encoded) > MAX_GOSSIP_RESPONSE_BYTES: + raise ValueError("checkpoint gossip response exceeds protocol limit") + writer.write(encoded) + await writer.drain() + + def _write_descriptor(self, port: int) -> None: + path = self.descriptor_path + path.parent.mkdir(parents=True, exist_ok=True) + path.parent.chmod(0o700) + if path.exists(): + raise ValueError("checkpoint gossip descriptor already exists") + descriptor = CheckpointGossipDescriptor( + protocol_version=GOSSIP_PROTOCOL_VERSION, + host=GOSSIP_LOOPBACK_HOST, + port=port, + registry_id=self.registry_id, + source_peer_id=self.source_peer_id, + instance_id=self._instance_id, + token=base64.b64encode(self._token).decode(), + ) + temporary = path.with_suffix(f".{self._instance_id[:12]}.tmp") + try: + temporary.write_text( + _canonical_json(descriptor.model_dump(mode="json")) + "\n", + encoding="utf-8", + ) + temporary.chmod(0o600) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + path.chmod(0o600) + + def _remove_owned_descriptor(self) -> None: + try: + payload = json.loads(self.descriptor_path.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): + return + if not isinstance(payload, dict): + return + if ( + payload.get("instance_id") == self._instance_id + and payload.get("token") == base64.b64encode(self._token).decode() + ): + self.descriptor_path.unlink(missing_ok=True) + + +class GossipRequestError(ValueError): + """A safe protocol error code returned by a checkpoint gossip handler.""" + + def __init__(self, code: str) -> None: + _validate_identifier(code, label="checkpoint gossip error code") + self.code = code + super().__init__(code) + + +async def send_checkpoint_gossip_request( + *, + descriptor_path: Path, + operation: str, + payload: dict[str, object], + client_nonce: bytes | None = None, + request_timeout_seconds: float = DEFAULT_GOSSIP_REQUEST_TIMEOUT_SECONDS, +) -> dict[str, object]: + """Send one bounded request after a one-use authenticated nonce challenge.""" + _validate_identifier(operation, label="operation") + if not 0 < request_timeout_seconds <= 30: + raise ValueError("checkpoint gossip timeout must be in (0, 30] seconds") + descriptor = _read_validated_descriptor(descriptor_path) + token = _decode_base64( + descriptor.token, + label="checkpoint gossip token", + length=32, + ) + nonce_bytes = client_nonce if client_nonce is not None else secrets.token_bytes(32) + if len(nonce_bytes) != 32: + raise ValueError("checkpoint gossip client nonce must contain 32 bytes") + client_nonce_base64 = base64.b64encode(nonce_bytes).decode() + challenge = { + "client_nonce": client_nonce_base64, + "instance_id": descriptor.instance_id, + "phase": "challenge", + "protocol_version": GOSSIP_PROTOCOL_VERSION, + "registry_id": descriptor.registry_id, + "source_peer_id": descriptor.source_peer_id, + } + reader: asyncio.StreamReader + writer: asyncio.StreamWriter + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection( + descriptor.host, + descriptor.port, + limit=MAX_GOSSIP_RESPONSE_BYTES, + ), + timeout=request_timeout_seconds, + ) + except (OSError, TimeoutError) as exc: + raise RuntimeError("checkpoint gossip endpoint is unavailable") from exc + try: + await _client_write_request(writer, challenge) + challenge_response = await _client_read_response( + reader, + timeout_seconds=request_timeout_seconds, + ) + if challenge_response.get("ok") is not True: + raise RuntimeError( + "checkpoint gossip request rejected: " + f"{challenge_response.get('error', 'unknown_error')}" + ) + _verify_authenticated_response( + challenge_response, + token=token, + descriptor=descriptor, + client_nonce=client_nonce_base64, + ) + server_nonce = challenge_response.get("server_nonce") + if not isinstance(server_nonce, str): + raise RuntimeError("checkpoint gossip challenge is invalid") + _decode_base64( + server_nonce, + label="checkpoint gossip server nonce", + length=32, + ) + request_payload = { + "client_nonce": client_nonce_base64, + "instance_id": descriptor.instance_id, + "operation": operation, + "payload": payload, + "phase": "request", + "protocol_version": GOSSIP_PROTOCOL_VERSION, + "registry_id": descriptor.registry_id, + "server_nonce": server_nonce, + "source_peer_id": descriptor.source_peer_id, + } + authenticated_request = { + **request_payload, + "authentication_sha256": _authentication_sha256(token, request_payload), + } + await _client_write_request(writer, authenticated_request) + response = await _client_read_response( + reader, + timeout_seconds=request_timeout_seconds, + ) + _verify_authenticated_response( + response, + token=token, + descriptor=descriptor, + client_nonce=client_nonce_base64, + server_nonce=server_nonce, + ) + if response.get("ok") is not True: + raise RuntimeError( + "checkpoint gossip request rejected: " + f"{response.get('error', 'unknown_error')}" + ) + result = response.get("result") + if not isinstance(result, dict): + raise RuntimeError("checkpoint gossip endpoint returned an invalid result") + return result + finally: + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + + +def _read_validated_descriptor(path: Path) -> CheckpointGossipDescriptor: + mode = stat.S_IMODE(path.stat().st_mode) + if mode & 0o077: + raise ValueError("checkpoint gossip descriptor permissions must be 0600") + return CheckpointGossipDescriptor.model_validate_json(path.read_text(encoding="utf-8")) + + +async def _client_write_request( + writer: asyncio.StreamWriter, + payload: dict[str, object], +) -> None: + encoded = _canonical_json(payload).encode("utf-8") + b"\n" + if len(encoded) > MAX_GOSSIP_REQUEST_BYTES: + raise ValueError("checkpoint gossip request exceeds protocol request limit") + writer.write(encoded) + await writer.drain() + + +async def _client_read_response( + reader: asyncio.StreamReader, + *, + timeout_seconds: float, +) -> dict[str, Any]: + raw = await asyncio.wait_for(reader.readline(), timeout=timeout_seconds) + if not raw or len(raw) > MAX_GOSSIP_RESPONSE_BYTES or not raw.endswith(b"\n"): + raise RuntimeError("checkpoint gossip endpoint returned an invalid response") + response = json.loads(raw.decode("utf-8")) + if not isinstance(response, dict) or not isinstance(response.get("ok"), bool): + raise RuntimeError("checkpoint gossip endpoint returned an invalid response") + return response + + +def _verify_authenticated_response( + response: dict[str, Any], + *, + token: bytes, + descriptor: CheckpointGossipDescriptor, + client_nonce: str, + server_nonce: str | None = None, +) -> None: + authentication = response.get("authentication_sha256") + if not isinstance(authentication, str): + raise RuntimeError("checkpoint gossip response authentication is missing") + binding = { + key: value + for key, value in response.items() + if key != "authentication_sha256" + } + expected = _authentication_sha256(token, binding) + if not hmac.compare_digest(authentication, expected): + raise RuntimeError("checkpoint gossip response authentication is invalid") + if ( + response.get("protocol_version") != GOSSIP_PROTOCOL_VERSION + or response.get("instance_id") != descriptor.instance_id + or response.get("client_nonce") != client_nonce + or response.get("registry_id") != descriptor.registry_id + or response.get("source_peer_id") != descriptor.source_peer_id + ): + raise RuntimeError("checkpoint gossip response challenge does not match") + if server_nonce is not None and response.get("server_nonce") != server_nonce: + raise RuntimeError("checkpoint gossip response server nonce does not match") diff --git a/benchmarks/agent_cli_peer_trust_ledger.py b/benchmarks/agent_cli_peer_trust_ledger.py new file mode 100644 index 0000000..aa5edbc --- /dev/null +++ b/benchmarks/agent_cli_peer_trust_ledger.py @@ -0,0 +1,462 @@ +"""Versioned peer-trust rollover ledger for authenticated checkpoint sync.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from typing import Literal + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerKey, + CheckpointPeerTrust, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode()).hexdigest() + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if not identifier or identifier != identifier.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +class CheckpointPeerTrustRotationStatement(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + generation: int = Field(ge=2) + predecessor_peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + successor_peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + minimum_distinct_peer_signatures: int = Field(ge=1) + rotation_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> CheckpointPeerTrustRotationStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + if self.predecessor_peer_trust_sha256 == self.successor_peer_trust_sha256: + raise ValueError("peer trust rotation must change peer trust") + if self.rotation_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("peer trust rotation fingerprint does not match statement") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"rotation_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class CheckpointPeerTrustRotationSigningRequest(_FrozenModel): + peer_id: str = Field(min_length=1, max_length=200) + eligible_key_ids: tuple[str, ...] = Field(min_length=1) + statement: CheckpointPeerTrustRotationStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_request(self) -> CheckpointPeerTrustRotationSigningRequest: + _validate_identifier(self.peer_id, label="peer_id") + if tuple(sorted(set(self.eligible_key_ids))) != self.eligible_key_ids: + raise ValueError("eligible peer rotation key IDs must be sorted and unique") + expected = self.statement.signing_bytes() + decoded = _decode_base64( + self.signing_payload_base64, + label="peer trust rotation signing payload", + length=len(expected), + ) + if decoded != expected: + raise ValueError("peer trust rotation payload does not match statement") + return self + + +class CheckpointPeerTrustRotationTemplate(_FrozenModel): + schema_version: int + generation: int = Field(ge=2) + predecessor_peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + successor_peer_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + minimum_distinct_peer_signatures: int = Field(ge=1) + statement: CheckpointPeerTrustRotationStatement + requests: tuple[CheckpointPeerTrustRotationSigningRequest, ...] = Field( + min_length=1 + ) + signatures_completed: Literal[False] = False + + @model_validator(mode="after") + def validate_template(self) -> CheckpointPeerTrustRotationTemplate: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.generation != self.statement.generation: + raise ValueError("peer trust rotation template generation does not match") + if self.predecessor_peer_trust_sha256 != ( + self.statement.predecessor_peer_trust_sha256 + ): + raise ValueError("peer trust rotation template predecessor does not match") + if self.successor_peer_trust_sha256 != ( + self.statement.successor_peer_trust_sha256 + ): + raise ValueError("peer trust rotation template successor does not match") + if self.minimum_distinct_peer_signatures != ( + self.statement.minimum_distinct_peer_signatures + ): + raise ValueError("peer trust rotation template quorum does not match") + if tuple(sorted(self.requests, key=lambda item: item.peer_id)) != self.requests: + raise ValueError("peer trust rotation requests must be sorted") + peer_ids = [request.peer_id for request in self.requests] + if len(peer_ids) != len(set(peer_ids)): + raise ValueError("peer trust rotation requests must use distinct peers") + if any(request.statement != self.statement for request in self.requests): + raise ValueError("peer trust rotation requests use different statements") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class CheckpointPeerRotationSignature(_FrozenModel): + peer_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature(self) -> CheckpointPeerRotationSignature: + _validate_identifier(self.peer_id, label="peer_id") + _validate_identifier(self.key_id, label="key_id") + _decode_base64( + self.signature_base64, + label="peer trust rotation signature", + length=64, + ) + return self + + +def _signature_identity( + signature: CheckpointPeerRotationSignature, +) -> tuple[str, str]: + return signature.peer_id, signature.key_id + + +class CheckpointPeerTrustRotationCertificate(_FrozenModel): + schema_version: int + statement: CheckpointPeerTrustRotationStatement + signatures: tuple[CheckpointPeerRotationSignature, ...] = Field(min_length=1) + rotation_certificate_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_certificate(self) -> CheckpointPeerTrustRotationCertificate: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if tuple(sorted(self.signatures, key=_signature_identity)) != self.signatures: + raise ValueError("peer trust rotation signatures must be sorted") + peer_ids = [signature.peer_id for signature in self.signatures] + if len(peer_ids) != len(set(peer_ids)): + raise ValueError("peer trust rotation signatures must use distinct peers") + if self.rotation_certificate_sha256 != _canonical_sha256( + self._binding_payload() + ): + raise ValueError("peer trust rotation certificate fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"rotation_certificate_sha256"}) + + +class CheckpointPeerTrustGeneration(_FrozenModel): + schema_version: int + generation: int = Field(ge=1) + trust: CheckpointPeerTrust + rotation: CheckpointPeerTrustRotationCertificate | None = None + + @model_validator(mode="after") + def validate_generation(self) -> CheckpointPeerTrustGeneration: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if (self.generation == 1) != (self.rotation is None): + raise ValueError("only genesis peer trust may omit a rotation certificate") + return self + + +class CheckpointPeerTrustLedger(_FrozenModel): + schema_version: int + registry_id: str = Field(min_length=1, max_length=200) + generations: tuple[CheckpointPeerTrustGeneration, ...] = Field(min_length=1) + active_generation: int = Field(ge=1) + ledger_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_ledger(self) -> CheckpointPeerTrustLedger: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.registry_id, label="registry_id") + _validate_generation_chain(self.generations) + if any( + generation.trust.registry_id != self.registry_id + for generation in self.generations + ): + raise ValueError("peer trust ledger generations use another registry") + if self.active_generation != self.generations[-1].generation: + raise ValueError("active_generation must identify the latest peer trust") + if self.ledger_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("peer trust ledger fingerprint does not match generations") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"ledger_sha256"}) + + @property + def active_trust(self) -> CheckpointPeerTrust: + return self.generations[-1].trust + + def resolve_peer_trust(self, peer_trust_sha256: str) -> CheckpointPeerTrust: + for generation in self.generations: + if generation.trust.peer_trust_sha256 == peer_trust_sha256: + return generation.trust + raise ValueError( + f"peer trust {peer_trust_sha256} is not present in peer trust ledger" + ) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _active_keys_by_peer( + trust: CheckpointPeerTrust, +) -> dict[str, tuple[CheckpointPeerKey, ...]]: + grouped: dict[str, list[CheckpointPeerKey]] = {} + for key in trust.keys: + if key.status == "active": + grouped.setdefault(key.peer_id, []).append(key) + return { + peer_id: tuple(sorted(keys, key=lambda key: key.key_id)) + for peer_id, keys in grouped.items() + } + + +def _rotation_statement( + predecessor: CheckpointPeerTrust, + successor: CheckpointPeerTrust, + *, + generation: int, +) -> CheckpointPeerTrustRotationStatement: + predecessor = CheckpointPeerTrust.model_validate( + predecessor.model_dump(mode="json") + ) + successor = CheckpointPeerTrust.model_validate(successor.model_dump(mode="json")) + if generation < 2: + raise ValueError("peer trust rotation generation must be at least 2") + if predecessor.registry_id != successor.registry_id: + raise ValueError("peer trust rotation changes registry_id") + if predecessor.peer_trust_sha256 == successor.peer_trust_sha256: + raise ValueError("peer trust rotation must change peer trust") + active_peers = _active_keys_by_peer(predecessor) + minimum = len(active_peers) // 2 + 1 + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": predecessor.registry_id, + "generation": generation, + "predecessor_peer_trust_sha256": predecessor.peer_trust_sha256, + "successor_peer_trust_sha256": successor.peer_trust_sha256, + "minimum_distinct_peer_signatures": minimum, + } + return CheckpointPeerTrustRotationStatement.model_validate( + {**payload, "rotation_sha256": _canonical_sha256(payload)} + ) + + +def build_checkpoint_peer_trust_rotation_template( + predecessor: CheckpointPeerTrust, + successor: CheckpointPeerTrust, + *, + generation: int, +) -> CheckpointPeerTrustRotationTemplate: + """Create one detached signing request per active predecessor peer.""" + statement = _rotation_statement( + predecessor, + successor, + generation=generation, + ) + requests = tuple( + CheckpointPeerTrustRotationSigningRequest( + peer_id=peer_id, + eligible_key_ids=tuple(key.key_id for key in keys), + statement=statement, + signing_payload_base64=base64.b64encode( + statement.signing_bytes() + ).decode(), + ) + for peer_id, keys in sorted(_active_keys_by_peer(predecessor).items()) + ) + return CheckpointPeerTrustRotationTemplate( + schema_version=SCHEMA_VERSION, + generation=generation, + predecessor_peer_trust_sha256=predecessor.peer_trust_sha256, + successor_peer_trust_sha256=successor.peer_trust_sha256, + minimum_distinct_peer_signatures=( + statement.minimum_distinct_peer_signatures + ), + statement=statement, + requests=requests, + ) + + +def verify_checkpoint_peer_trust_rotation_certificate( + certificate: CheckpointPeerTrustRotationCertificate, + predecessor: CheckpointPeerTrust, + successor: CheckpointPeerTrust, + *, + generation: int, +) -> None: + """Verify one strict-majority rollover against the predecessor key set.""" + certificate = CheckpointPeerTrustRotationCertificate.model_validate( + certificate.model_dump(mode="json") + ) + expected = _rotation_statement( + predecessor, + successor, + generation=generation, + ) + if certificate.statement != expected: + raise ValueError("peer trust rotation certificate does not match generation") + if len(certificate.signatures) < expected.minimum_distinct_peer_signatures: + raise ValueError("peer trust rotation strict-majority quorum is incomplete") + active_keys = { + (key.peer_id, key.key_id): key + for key in predecessor.keys + if key.status == "active" + } + for signature in certificate.signatures: + key = active_keys.get(_signature_identity(signature)) + if key is None: + raise ValueError("peer trust rotation signature is not an active predecessor key") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64( + key.public_key_base64, + label="peer public key", + length=32, + ) + ).verify( + _decode_base64( + signature.signature_base64, + label="peer trust rotation signature", + length=64, + ), + certificate.statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError( + f"peer trust rotation signature is invalid: {signature.peer_id}" + ) from exc + + +def build_checkpoint_peer_trust_rotation_certificate( + template: CheckpointPeerTrustRotationTemplate, + predecessor: CheckpointPeerTrust, + successor: CheckpointPeerTrust, + signatures: tuple[CheckpointPeerRotationSignature, ...], +) -> CheckpointPeerTrustRotationCertificate: + """Normalize and verify detached strict-majority rollover signatures.""" + expected_template = build_checkpoint_peer_trust_rotation_template( + predecessor, + successor, + generation=template.generation, + ) + if template != expected_template: + raise ValueError("peer trust rotation template does not match trusts") + ordered = tuple(sorted(signatures, key=_signature_identity)) + payload = { + "schema_version": SCHEMA_VERSION, + "statement": template.statement.model_dump(mode="json"), + "signatures": [signature.model_dump(mode="json") for signature in ordered], + } + certificate = CheckpointPeerTrustRotationCertificate.model_validate( + { + **payload, + "rotation_certificate_sha256": _canonical_sha256(payload), + } + ) + verify_checkpoint_peer_trust_rotation_certificate( + certificate, + predecessor, + successor, + generation=template.generation, + ) + return certificate + + +def _validate_generation_chain( + generations: tuple[CheckpointPeerTrustGeneration, ...], +) -> None: + if not generations: + raise ValueError("peer trust ledger requires at least one generation") + fingerprints: set[str] = set() + registry_id = generations[0].trust.registry_id + for index, generation in enumerate(generations): + if generation.generation != index + 1: + raise ValueError("peer trust generations must be contiguous and ordered") + if generation.trust.registry_id != registry_id: + raise ValueError("peer trust generations change registry_id") + fingerprint = generation.trust.peer_trust_sha256 + if fingerprint in fingerprints: + raise ValueError("peer trust generations must not reuse peer trust") + fingerprints.add(fingerprint) + if index == 0: + if generation.rotation is not None: + raise ValueError("genesis peer trust must not have a rotation") + continue + rotation = generation.rotation + if rotation is None: + raise ValueError("peer trust generation is missing its rotation") + verify_checkpoint_peer_trust_rotation_certificate( + rotation, + generations[index - 1].trust, + generation.trust, + generation=generation.generation, + ) + + +def build_checkpoint_peer_trust_ledger( + generations: tuple[CheckpointPeerTrustGeneration, ...], +) -> CheckpointPeerTrustLedger: + """Verify and self-fingerprint a complete peer-trust generation chain.""" + _validate_generation_chain(generations) + payload = { + "schema_version": SCHEMA_VERSION, + "registry_id": generations[0].trust.registry_id, + "generations": [generation.model_dump(mode="json") for generation in generations], + "active_generation": generations[-1].generation, + } + return CheckpointPeerTrustLedger.model_validate( + {**payload, "ledger_sha256": _canonical_sha256(payload)} + ) diff --git a/benchmarks/agent_cli_preflight.py b/benchmarks/agent_cli_preflight.py new file mode 100644 index 0000000..73196f8 --- /dev/null +++ b/benchmarks/agent_cli_preflight.py @@ -0,0 +1,301 @@ +"""Offline campaign preflight and evidence-bound review template generation.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_adjudication import ( + AdjudicationReviews, + RecordedEvidence, + RecordedTrialEvidence, + recorded_evidence_sha256, +) +from benchmarks.agent_cli_comparison import ( + REQUIRED_ARMS, + SCHEMA_VERSION, + AgentCliArm, + AgentCliManifest, +) +from benchmarks.agent_cli_recorder import ( + AgentCliRecorderConfig, + build_recording_plan, +) +from benchmarks.agent_cli_rehearsal import resolve_git_revision as _resolve_git_revision + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" +_COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") + + +async def resolve_git_revision(source_root: Path, revision: str) -> str: + """Resolve the campaign revision through the shared read-only Git helper.""" + return await _resolve_git_revision(source_root, revision) + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +class RuntimeVersionDeclaration(_FrozenModel): + executable: str = Field(min_length=1, max_length=1024) + version: str = Field(min_length=1, max_length=500) + + +class RuntimeVersionBundle(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + runtimes: dict[AgentCliArm, RuntimeVersionDeclaration] + + @model_validator(mode="after") + def validate_contract(self) -> RuntimeVersionBundle: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if set(self.runtimes) != REQUIRED_ARMS: + raise ValueError("runtimes must contain exactly the three comparison arms") + return self + + +class RuntimeFingerprint(_FrozenModel): + executable: str = Field(min_length=1, max_length=1024) + version: str = Field(min_length=1, max_length=500) + version_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_fingerprint(self) -> RuntimeFingerprint: + if self.version != _normalize_version(self.version): + raise ValueError("runtime version must be whitespace-normalized") + expected = hashlib.sha256(self.version.encode()).hexdigest() + if self.version_sha256 != expected: + raise ValueError("runtime version fingerprint does not match version") + return self + + +def _normalize_version(version: str) -> str: + normalized = " ".join(version.split()) + if not normalized: + raise ValueError("runtime version must not be blank") + return normalized + + +def _canonical_sha256(payload: object) -> str: + body = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(body.encode()).hexdigest() + + +class CampaignPreflight(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + workspace_revision: str = Field(pattern=r"^[0-9a-f]{40}$") + arms: tuple[AgentCliArm, ...] + repetitions: int = Field(ge=1) + trial_count: int = Field(ge=1) + estimated_max_cost_usd: float = Field(ge=0.0) + manifest_sha256: str = Field(pattern=_SHA256_PATTERN) + config_sha256: str = Field(pattern=_SHA256_PATTERN) + command_fingerprints: dict[str, str] + runtime_fingerprints: dict[AgentCliArm, RuntimeFingerprint] + execution_authorized: Literal[False] = False + preflight_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_contract(self) -> CampaignPreflight: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if len(self.arms) != len(REQUIRED_ARMS) or set(self.arms) != REQUIRED_ARMS: + raise ValueError("preflight arms must contain exactly the comparison arms") + if set(self.runtime_fingerprints) != REQUIRED_ARMS: + raise ValueError("runtime fingerprints must contain exactly the comparison arms") + if self.trial_count != len(self.arms) * self.repetitions: + raise ValueError("trial_count does not match arms and repetitions") + if not self.command_fingerprints or any( + re.fullmatch(_SHA256_PATTERN, fingerprint) is None + for fingerprint in self.command_fingerprints.values() + ): + raise ValueError("command fingerprints must be non-empty SHA-256 values") + expected = _canonical_sha256(self._binding_payload()) + if self.preflight_sha256 != expected: + raise ValueError("preflight fingerprint does not match report") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"preflight_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_campaign_preflight( + manifest: AgentCliManifest, + config: AgentCliRecorderConfig, + versions: RuntimeVersionBundle, + *, + resolved_revision: str, +) -> CampaignPreflight: + """Validate a fully pinned campaign without authorizing or launching agents.""" + if not _COMMIT_PATTERN.fullmatch(resolved_revision): + raise ValueError("resolved revision must be a full lowercase Git commit") + if manifest.task.workspace_revision != resolved_revision: + raise ValueError("manifest must pin the immutable resolved revision") + if versions.benchmark_id != manifest.benchmark_id: + raise ValueError("runtime versions benchmark_id does not match manifest") + plan = build_recording_plan(manifest, config) + runtime_fingerprints: dict[AgentCliArm, RuntimeFingerprint] = {} + for arm in manifest.arms: + declaration = versions.runtimes[arm] + expected_executable = config.arm_commands[arm][0] + if declaration.executable != expected_executable: + raise ValueError(f"runtime executable does not match {arm.value} command") + normalized_version = _normalize_version(declaration.version) + runtime_fingerprints[arm] = RuntimeFingerprint( + executable=declaration.executable, + version=normalized_version, + version_sha256=hashlib.sha256(normalized_version.encode()).hexdigest(), + ) + payload = { + "schema_version": SCHEMA_VERSION, + "benchmark_id": manifest.benchmark_id, + "task_id": manifest.task.id, + "workspace_revision": resolved_revision, + "arms": [arm.value for arm in manifest.arms], + "repetitions": manifest.repetitions, + "trial_count": plan.trial_count, + "estimated_max_cost_usd": plan.estimated_max_cost_usd, + "manifest_sha256": _canonical_sha256(manifest.model_dump(mode="json")), + "config_sha256": _canonical_sha256(config.model_dump(mode="json")), + "command_fingerprints": plan.command_fingerprints, + "runtime_fingerprints": { + arm.value: fingerprint.model_dump(mode="json") + for arm, fingerprint in runtime_fingerprints.items() + }, + "execution_authorized": False, + } + return CampaignPreflight( + **payload, + preflight_sha256=_canonical_sha256(payload), + ) + + +class ReviewDecisionTemplate(_FrozenModel): + arm: AgentCliArm + trial: int = Field(ge=1) + agent_argv_sha256: str = Field(pattern=_SHA256_PATTERN) + accepted_patch: None = None + human_interventions: None = None + recovery_attempted: None = None + recovery_succeeded: None = None + reviewer_id: None = None + review_artifact_sha256: None = None + + +class ReviewTemplate(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + task_id: str = Field(min_length=1) + workspace_revision: str = Field(pattern=r"^[0-9a-f]{40}$") + preflight_sha256: str = Field(pattern=_SHA256_PATTERN) + evidence_sha256: str = Field(pattern=_SHA256_PATTERN) + review_policy_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + reviewer_trust_sha256: str | None = Field(default=None, pattern=_SHA256_PATTERN) + decisions: tuple[ReviewDecisionTemplate, ...] = Field(min_length=1) + review_completed: Literal[False] = False + + @model_validator(mode="after") + def validate_trust_binding(self) -> ReviewTemplate: + if self.reviewer_trust_sha256 is not None and self.review_policy_sha256 is None: + raise ValueError("reviewer trust binding requires a review policy binding") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _expected_cells(preflight: CampaignPreflight) -> set[tuple[AgentCliArm, int]]: + return { + (arm, trial) + for arm in preflight.arms + for trial in range(1, preflight.repetitions + 1) + } + + +def build_review_template( + preflight: CampaignPreflight, + evidence: RecordedEvidence, + *, + review_policy_sha256: str | None = None, + reviewer_trust_sha256: str | None = None, +) -> ReviewTemplate: + """Create null review decisions bound to exact preflight and evidence artifacts.""" + expected_identity = ( + preflight.benchmark_id, + preflight.task_id, + preflight.workspace_revision, + ) + if ( + evidence.benchmark_id, + evidence.task_id, + evidence.workspace_revision, + ) != expected_identity: + raise ValueError("evidence identity does not match preflight") + indexed: dict[tuple[AgentCliArm, int], RecordedTrialEvidence] = {} + for trial in evidence.trials: + key = (trial.arm, trial.trial) + if key in indexed: + raise ValueError(f"duplicate evidence for {trial.arm.value}:{trial.trial}") + indexed[key] = trial + expected = _expected_cells(preflight) + if set(indexed) != expected: + raise ValueError("evidence trial matrix does not match preflight") + decisions = tuple( + ReviewDecisionTemplate( + arm=arm, + trial=trial_number, + agent_argv_sha256=indexed[(arm, trial_number)].agent.argv_sha256, + ) + for arm in preflight.arms + for trial_number in range(1, preflight.repetitions + 1) + ) + return ReviewTemplate( + schema_version=SCHEMA_VERSION, + benchmark_id=preflight.benchmark_id, + task_id=preflight.task_id, + workspace_revision=preflight.workspace_revision, + preflight_sha256=preflight.preflight_sha256, + evidence_sha256=recorded_evidence_sha256(evidence), + review_policy_sha256=review_policy_sha256, + reviewer_trust_sha256=reviewer_trust_sha256, + decisions=decisions, + ) + + +def validate_review_bindings( + preflight: CampaignPreflight, + evidence: RecordedEvidence, + reviews: AdjudicationReviews, +) -> None: + """Require completed reviews to retain their generated artifact bindings.""" + if reviews.preflight_sha256 != preflight.preflight_sha256: + raise ValueError("review preflight fingerprint does not match preflight") + if reviews.evidence_sha256 != recorded_evidence_sha256(evidence): + raise ValueError("review evidence fingerprint does not match evidence") + expected_identity = ( + preflight.benchmark_id, + preflight.task_id, + preflight.workspace_revision, + ) + if ( + reviews.benchmark_id, + reviews.task_id, + reviews.workspace_revision, + ) != expected_identity: + raise ValueError("review identity does not match preflight") diff --git a/benchmarks/agent_cli_receipts.py b/benchmarks/agent_cli_receipts.py new file mode 100644 index 0000000..5442215 --- /dev/null +++ b/benchmarks/agent_cli_receipts.py @@ -0,0 +1,181 @@ +"""Normalize provider output into privacy-preserving benchmark receipts.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_comparison import AgentCliArm +from domain.entities.council_runtime import CouncilTurn +from domain.services.engine_cost_calculator import EngineCostCalculator +from infrastructure.agent_cli.claude_jsonl import parse_claude_output +from infrastructure.agent_cli.codex_jsonl import parse_codex_output + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +class ProviderReceipt(_FrozenModel): + """Normalized cost/success evidence with provider output removed.""" + + provider: AgentCliArm + success: bool + model: str = Field(min_length=1) + usage: dict[str, int] + cost_usd: float = Field(ge=0.0) + cost_source: Literal[ + "provider_reported", + "calculated_from_usage", + "morphic_reported", + ] + parse_errors: int = Field(default=0, ge=0) + + @model_validator(mode="after") + def validate_receipt(self) -> ProviderReceipt: + if any(tokens < 0 for tokens in self.usage.values()): + raise ValueError("usage token counts must be non-negative") + expected_sources = { + AgentCliArm.CODEX_CLI: "calculated_from_usage", + AgentCliArm.CLAUDE_CODE: "provider_reported", + AgentCliArm.MORPHIC_CONTROL: "morphic_reported", + } + if self.cost_source != expected_sources[self.provider]: + raise ValueError("cost_source does not match provider") + if self.provider is AgentCliArm.CODEX_CLI: + if not self.usage: + raise ValueError("Codex calculated receipt requires usage") + calculated = EngineCostCalculator.calculate(self.model, self.usage) + if abs(self.cost_usd - calculated) > 0.000001: + raise ValueError("Codex receipt cost does not match usage") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class MorphicBenchmarkReceiptEnvelope(_FrozenModel): + """Canonical final stdout line emitted by Morphic benchmark runs.""" + + type: Literal["morphic_benchmark_receipt"] + success: bool + model: str = Field(min_length=1) + usage: dict[str, int] + cost_usd: float = Field(ge=0.0) + + @model_validator(mode="after") + def validate_usage(self) -> MorphicBenchmarkReceiptEnvelope: + if any(tokens < 0 for tokens in self.usage.values()): + raise ValueError("usage token counts must be non-negative") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_morphic_benchmark_receipt( + turns: Iterable[CouncilTurn], + *, + success: bool = True, +) -> MorphicBenchmarkReceiptEnvelope: + """Aggregate privacy-safe usage and cost from one Morphic council run.""" + materialized = tuple(turns) + usage: dict[str, int] = {} + for turn in materialized: + for event in turn.engine_events: + raw_usage = event.payload.get("usage") + if not isinstance(raw_usage, dict): + continue + for name, tokens in raw_usage.items(): + if ( + isinstance(name, str) + and isinstance(tokens, int) + and not isinstance(tokens, bool) + ): + if tokens < 0: + continue + usage[name] = usage.get(name, 0) + tokens + engines = sorted({turn.engine_id for turn in materialized}) + model = "morphic-control" + if engines: + model = f"morphic-control[{','.join(engines)}]" + return MorphicBenchmarkReceiptEnvelope( + type="morphic_benchmark_receipt", + success=success, + model=model, + usage=usage, + cost_usd=round(sum(turn.cost_usd for turn in materialized), 6), + ) + + +class ProviderReceiptParser: + """Parse supported provider outputs without retaining their raw content.""" + + def parse( + self, + *, + arm: AgentCliArm | str, + stdout: str, + model_hint: str | None = None, + ) -> ProviderReceipt | None: + parsed_arm = AgentCliArm(arm) + if parsed_arm is AgentCliArm.CODEX_CLI: + return self._parse_codex(stdout, model_hint=model_hint) + if parsed_arm is AgentCliArm.CLAUDE_CODE: + return self._parse_claude(stdout, model_hint=model_hint) + return self._parse_morphic(stdout) + + def _parse_codex(self, stdout: str, *, model_hint: str | None) -> ProviderReceipt | None: + parsed = parse_codex_output(stdout) + model = parsed.model or model_hint + if not parsed.usage or not model: + return None + return ProviderReceipt( + provider=AgentCliArm.CODEX_CLI, + success=parsed.error is None, + model=model, + usage=parsed.usage, + cost_usd=EngineCostCalculator.calculate(model, parsed.usage), + cost_source="calculated_from_usage", + parse_errors=parsed.parse_errors, + ) + + def _parse_claude(self, stdout: str, *, model_hint: str | None) -> ProviderReceipt | None: + parsed = parse_claude_output(stdout) + model = parsed.model or model_hint + if not model or (not parsed.usage and parsed.cost_usd == 0.0): + return None + return ProviderReceipt( + provider=AgentCliArm.CLAUDE_CODE, + success=parsed.error is None, + model=model, + usage=parsed.usage or {}, + cost_usd=parsed.cost_usd, + cost_source="provider_reported", + parse_errors=parsed.parse_errors, + ) + + def _parse_morphic(self, stdout: str) -> ProviderReceipt | None: + for line in reversed(stdout.splitlines()): + try: + raw = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(raw, dict) or raw.get("type") != "morphic_benchmark_receipt": + continue + try: + envelope = MorphicBenchmarkReceiptEnvelope.model_validate(raw) + except ValueError: + return None + return ProviderReceipt( + provider=AgentCliArm.MORPHIC_CONTROL, + success=envelope.success, + model=envelope.model, + usage=envelope.usage, + cost_usd=envelope.cost_usd, + cost_source="morphic_reported", + ) + return None diff --git a/benchmarks/agent_cli_recorder.py b/benchmarks/agent_cli_recorder.py new file mode 100644 index 0000000..f9831c4 --- /dev/null +++ b/benchmarks/agent_cli_recorder.py @@ -0,0 +1,587 @@ +"""Explicit opt-in recorder for isolated same-task agent CLI trials. + +Dry-run planning is pure and deterministic. Live execution requires a separate +paid-run acknowledgement and an explicit cost cap. Each trial runs in a detached +Git worktree, and evidence stores hashes and byte counts instead of raw output. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import math +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Protocol + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_comparison import ( + REQUIRED_ARMS, + SCHEMA_VERSION, + AgentCliArm, + AgentCliManifest, +) +from benchmarks.agent_cli_receipts import ProviderReceipt, ProviderReceiptParser + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _validate_command_map(name: str, commands: dict[Any, tuple[str, ...]]) -> None: + for key, argv in commands.items(): + if not argv: + raise ValueError(f"{name}[{key}] must not be empty") + if any(not argument or "\x00" in argument for argument in argv): + raise ValueError(f"{name}[{key}] arguments must be non-empty and NUL-free") + + +class AgentCliRecorderConfig(_FrozenModel): + """Commands and safety estimates for one recorded benchmark.""" + + schema_version: int + benchmark_id: str = Field(min_length=1) + arm_commands: dict[AgentCliArm, tuple[str, ...]] + check_commands: dict[str, tuple[str, ...]] + handoff_commands: dict[str, tuple[str, ...]] + estimated_cost_usd_per_trial: dict[AgentCliArm, float] + model_hints: dict[AgentCliArm, str] = Field(default_factory=dict) + timeout_seconds: float = Field(gt=0.0, le=3600.0) + + @model_validator(mode="after") + def validate_config(self) -> AgentCliRecorderConfig: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if set(self.arm_commands) != REQUIRED_ARMS: + raise ValueError("arm_commands must contain exactly the three comparison arms") + if set(self.estimated_cost_usd_per_trial) != REQUIRED_ARMS: + raise ValueError( + "estimated_cost_usd_per_trial must contain exactly the three comparison arms" + ) + if any(value < 0.0 for value in self.estimated_cost_usd_per_trial.values()): + raise ValueError("estimated costs must be non-negative") + if not set(self.model_hints).issubset(REQUIRED_ARMS): + raise ValueError("model_hints contains an unsupported arm") + if any(not model.strip() for model in self.model_hints.values()): + raise ValueError("model_hints values must not be blank") + _validate_command_map("arm_commands", self.arm_commands) + _validate_command_map("check_commands", self.check_commands) + _validate_command_map("handoff_commands", self.handoff_commands) + return self + + +def _argv_sha256(argv: tuple[str, ...]) -> str: + body = json.dumps(argv, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(body.encode()).hexdigest() + + +@dataclass(frozen=True) +class RecordingPlan: + """Deterministic preview; it contains no raw task prompt.""" + + schema_version: int + benchmark_id: str + task_id: str + workspace_revision: str + repetitions: int + arms: list[str] + trial_count: int + estimated_max_cost_usd: float + timeout_seconds: float + command_fingerprints: dict[str, str] + execution_required: bool = False + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True) + + +def build_recording_plan( + manifest: AgentCliManifest, + config: AgentCliRecorderConfig, +) -> RecordingPlan: + """Validate recorder coverage and return a non-executing preview.""" + if config.benchmark_id != manifest.benchmark_id: + raise ValueError("recorder benchmark_id does not match manifest") + if set(config.arm_commands) != set(manifest.arms): + raise ValueError("arm_commands do not match manifest arms") + if set(config.check_commands) != set(manifest.task.checks): + raise ValueError("check_commands do not match manifest checks") + if set(config.handoff_commands) != set(manifest.task.handoff_assertions): + raise ValueError("handoff_commands do not match manifest handoff_assertions") + + fingerprints = { + **{ + f"arm:{arm.value}": _argv_sha256(config.arm_commands[arm]) + for arm in manifest.arms + }, + **{ + f"check:{name}": _argv_sha256(config.check_commands[name]) + for name in sorted(config.check_commands) + }, + **{ + f"handoff:{name}": _argv_sha256(config.handoff_commands[name]) + for name in sorted(config.handoff_commands) + }, + } + estimated_cost = sum( + config.estimated_cost_usd_per_trial[arm] * manifest.repetitions + for arm in manifest.arms + ) + return RecordingPlan( + schema_version=SCHEMA_VERSION, + benchmark_id=manifest.benchmark_id, + task_id=manifest.task.id, + workspace_revision=manifest.task.workspace_revision, + repetitions=manifest.repetitions, + arms=[arm.value for arm in manifest.arms], + trial_count=len(manifest.arms) * manifest.repetitions, + estimated_max_cost_usd=round(estimated_cost, 6), + timeout_seconds=config.timeout_seconds, + command_fingerprints=fingerprints, + ) + + +def validate_execution_consent( + plan: RecordingPlan, + *, + acknowledged_paid: bool, + cost_cap_usd: float | None, +) -> None: + """Fail before filesystem mutation unless live-run consent is complete.""" + if not acknowledged_paid: + raise ValueError("paid execution acknowledgement is required") + if cost_cap_usd is None or not math.isfinite(cost_cap_usd) or cost_cap_usd < 0.0: + raise ValueError("a finite non-negative cost cap is required") + if cost_cap_usd < plan.estimated_max_cost_usd: + raise ValueError( + "cost cap is below estimated maximum " + f"(${cost_cap_usd:.6f} < ${plan.estimated_max_cost_usd:.6f})" + ) + + +@dataclass(frozen=True) +class CommandCapture: + """Ephemeral raw command result returned by a runner.""" + + exit_code: int + stdout: str + stderr: str + elapsed_seconds: float + timed_out: bool + + +@dataclass(frozen=True) +class CommandEvidence: + """Persistable command evidence with raw output removed.""" + + argv_sha256: str + exit_code: int + timed_out: bool + elapsed_seconds: float + stdout_sha256: str + stdout_bytes: int + stderr_sha256: str + stderr_bytes: int + + @classmethod + def from_capture( + cls, + *, + argv: tuple[str, ...], + capture: CommandCapture, + ) -> CommandEvidence: + stdout = capture.stdout.encode() + stderr = capture.stderr.encode() + return cls( + argv_sha256=_argv_sha256(argv), + exit_code=capture.exit_code, + timed_out=capture.timed_out, + elapsed_seconds=round(capture.elapsed_seconds, 6), + stdout_sha256=hashlib.sha256(stdout).hexdigest(), + stdout_bytes=len(stdout), + stderr_sha256=hashlib.sha256(stderr).hexdigest(), + stderr_bytes=len(stderr), + ) + + +@dataclass(frozen=True) +class TrialEvidence: + arm: str + trial: int + reserved_cost_usd: float + agent: CommandEvidence + checks: dict[str, CommandEvidence] + handoff_assertions: dict[str, CommandEvidence] + receipt: ProviderReceipt | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "arm": self.arm, + "trial": self.trial, + "reserved_cost_usd": self.reserved_cost_usd, + "agent": asdict(self.agent), + "checks": {name: asdict(value) for name, value in self.checks.items()}, + "handoff_assertions": { + name: asdict(value) for name, value in self.handoff_assertions.items() + }, + "receipt": ( + self.receipt.model_dump(mode="json") if self.receipt is not None else None + ), + "completed": self.agent.exit_code == 0 and not self.agent.timed_out, + "passed_checks": [ + name + for name, value in self.checks.items() + if value.exit_code == 0 and not value.timed_out + ], + "passed_handoff_assertions": [ + name + for name, value in self.handoff_assertions.items() + if value.exit_code == 0 and not value.timed_out + ], + } + + +@dataclass(frozen=True) +class RecordingEvidence: + schema_version: int + benchmark_id: str + task_id: str + workspace_revision: str + estimated_max_cost_usd: float + authorized_cost_cap_usd: float + trials: list[TrialEvidence] + + def to_dict(self) -> dict[str, Any]: + cost_collection = ( + "normalized_receipts" + if all(trial.receipt is not None for trial in self.trials) + else "pending_adjudication" + ) + return { + "schema_version": self.schema_version, + "benchmark_id": self.benchmark_id, + "task_id": self.task_id, + "workspace_revision": self.workspace_revision, + "estimated_max_cost_usd": self.estimated_max_cost_usd, + "authorized_cost_cap_usd": self.authorized_cost_cap_usd, + "cost_collection": cost_collection, + "trials": [trial.to_dict() for trial in self.trials], + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True) + + +class WorktreeManagerPort(Protocol): + async def create(self, *, source_root: Path, revision: str, destination: Path) -> None: ... + + async def release(self, *, source_root: Path, destination: Path) -> None: ... + + +class CommandRunnerPort(Protocol): + async def run( + self, + *, + argv: tuple[str, ...], + cwd: Path, + timeout_seconds: float, + ) -> CommandCapture: ... + + +class ReceiptParserPort(Protocol): + def parse( + self, + *, + arm: AgentCliArm | str, + stdout: str, + model_hint: str | None = None, + ) -> ProviderReceipt | None: ... + + +def _expand_argv( + template: tuple[str, ...], + *, + manifest: AgentCliManifest, + arm: AgentCliArm, + trial: int, + workspace: Path, +) -> tuple[str, ...]: + replacements = { + "{goal}": manifest.task.goal, + "{workspace}": str(workspace), + "{arm}": arm.value, + "{trial}": str(trial), + } + return tuple( + argument.replace("{goal}", replacements["{goal}"]) + .replace("{workspace}", replacements["{workspace}"]) + .replace("{arm}", replacements["{arm}"]) + .replace("{trial}", replacements["{trial}"]) + for argument in template + ) + + +class AgentCliTrialRecorder: + """Run a complete trial matrix through injected isolation and command ports.""" + + def __init__( + self, + *, + worktree_manager: WorktreeManagerPort, + command_runner: CommandRunnerPort, + receipt_parser: ReceiptParserPort | None = None, + ) -> None: + self._worktree_manager = worktree_manager + self._command_runner = command_runner + self._receipt_parser = receipt_parser or ProviderReceiptParser() + + async def record( + self, + *, + manifest: AgentCliManifest, + config: AgentCliRecorderConfig, + source_root: Path, + worktree_root: Path, + acknowledged_paid: bool, + cost_cap_usd: float | None, + ) -> RecordingEvidence: + plan = build_recording_plan(manifest, config) + validate_execution_consent( + plan, + acknowledged_paid=acknowledged_paid, + cost_cap_usd=cost_cap_usd, + ) + assert cost_cap_usd is not None + source_root = source_root.resolve() + worktree_root = worktree_root.resolve() + if worktree_root == source_root or worktree_root.is_relative_to(source_root): + raise ValueError("worktree_root must be outside source_root") + worktree_root.mkdir(mode=0o700, parents=True, exist_ok=True) + trials: list[TrialEvidence] = [] + prefix = hashlib.sha256(manifest.benchmark_id.encode()).hexdigest()[:12] + + for arm in manifest.arms: + for trial in range(1, manifest.repetitions + 1): + destination = worktree_root / f"{prefix}-{arm.value}-{trial}" + if destination.exists(): + raise ValueError(f"worktree destination already exists: {destination}") + created = False + try: + await self._worktree_manager.create( + source_root=source_root, + revision=manifest.task.workspace_revision, + destination=destination, + ) + created = True + agent, receipt = await self._run_agent_evidence( + config.arm_commands[arm], + manifest=manifest, + arm=arm, + trial=trial, + workspace=destination, + timeout_seconds=config.timeout_seconds, + model_hint=config.model_hints.get(arm), + ) + checks = { + name: await self._run_evidence( + config.check_commands[name], + manifest=manifest, + arm=arm, + trial=trial, + workspace=destination, + timeout_seconds=config.timeout_seconds, + ) + for name in manifest.task.checks + } + handoffs = { + name: await self._run_evidence( + config.handoff_commands[name], + manifest=manifest, + arm=arm, + trial=trial, + workspace=destination, + timeout_seconds=config.timeout_seconds, + ) + for name in manifest.task.handoff_assertions + } + trials.append( + TrialEvidence( + arm=arm.value, + trial=trial, + reserved_cost_usd=config.estimated_cost_usd_per_trial[arm], + agent=agent, + checks=checks, + handoff_assertions=handoffs, + receipt=receipt, + ) + ) + finally: + if created: + await self._worktree_manager.release( + source_root=source_root, + destination=destination, + ) + + return RecordingEvidence( + schema_version=SCHEMA_VERSION, + benchmark_id=manifest.benchmark_id, + task_id=manifest.task.id, + workspace_revision=manifest.task.workspace_revision, + estimated_max_cost_usd=plan.estimated_max_cost_usd, + authorized_cost_cap_usd=cost_cap_usd, + trials=trials, + ) + + async def _run_evidence( + self, + template: tuple[str, ...], + *, + manifest: AgentCliManifest, + arm: AgentCliArm, + trial: int, + workspace: Path, + timeout_seconds: float, + ) -> CommandEvidence: + argv = _expand_argv( + template, + manifest=manifest, + arm=arm, + trial=trial, + workspace=workspace, + ) + capture = await self._command_runner.run( + argv=argv, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + return CommandEvidence.from_capture(argv=argv, capture=capture) + + async def _run_agent_evidence( + self, + template: tuple[str, ...], + *, + manifest: AgentCliManifest, + arm: AgentCliArm, + trial: int, + workspace: Path, + timeout_seconds: float, + model_hint: str | None, + ) -> tuple[CommandEvidence, ProviderReceipt | None]: + argv = _expand_argv( + template, + manifest=manifest, + arm=arm, + trial=trial, + workspace=workspace, + ) + capture = await self._command_runner.run( + argv=argv, + cwd=workspace, + timeout_seconds=timeout_seconds, + ) + receipt = self._receipt_parser.parse( + arm=arm, + stdout=capture.stdout, + model_hint=model_hint, + ) + return CommandEvidence.from_capture(argv=argv, capture=capture), receipt + + +class LocalCommandRunner: + """Run one argv vector without invoking a shell.""" + + async def run( + self, + *, + argv: tuple[str, ...], + cwd: Path, + timeout_seconds: float, + ) -> CommandCapture: + started = time.monotonic() + try: + process = await asyncio.create_subprocess_exec( + *argv, + cwd=cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except OSError as exc: + return CommandCapture( + exit_code=127, + stdout="", + stderr=str(exc), + elapsed_seconds=time.monotonic() - started, + timed_out=False, + ) + try: + stdout, stderr = await asyncio.wait_for( + process.communicate(), + timeout=timeout_seconds, + ) + except asyncio.CancelledError: + await _terminate_process(process) + raise + except TimeoutError: + await _terminate_process(process) + return CommandCapture( + exit_code=-1, + stdout="", + stderr=f"command timed out after {timeout_seconds}s", + elapsed_seconds=time.monotonic() - started, + timed_out=True, + ) + return CommandCapture( + exit_code=process.returncode or 0, + stdout=stdout.decode(errors="replace"), + stderr=stderr.decode(errors="replace"), + elapsed_seconds=time.monotonic() - started, + timed_out=False, + ) + + +async def _terminate_process(process: asyncio.subprocess.Process) -> None: + if process.returncode is not None: + return + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=2.0) + except TimeoutError: + process.kill() + await process.wait() + + +class GitWorktreeManager: + """Create and remove detached Git worktrees at a pinned revision.""" + + async def create(self, *, source_root: Path, revision: str, destination: Path) -> None: + capture = await LocalCommandRunner().run( + argv=( + "git", + "-C", + str(source_root), + "worktree", + "add", + "--detach", + str(destination), + revision, + ), + cwd=source_root, + timeout_seconds=60.0, + ) + if capture.exit_code != 0: + raise RuntimeError(f"git worktree add failed: {capture.stderr[:500]}") + + async def release(self, *, source_root: Path, destination: Path) -> None: + capture = await LocalCommandRunner().run( + argv=("git", "-C", str(source_root), "worktree", "remove", "--force", str(destination)), + cwd=source_root, + timeout_seconds=60.0, + ) + if capture.exit_code != 0: + raise RuntimeError(f"git worktree remove failed: {capture.stderr[:500]}") diff --git a/benchmarks/agent_cli_rehearsal.py b/benchmarks/agent_cli_rehearsal.py new file mode 100644 index 0000000..3cca7fa --- /dev/null +++ b/benchmarks/agent_cli_rehearsal.py @@ -0,0 +1,237 @@ +"""Zero-cost rehearsal for the recorded agent CLI benchmark pipeline.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path + +from pydantic import BaseModel + +from benchmarks.agent_cli_adjudication import ( + AdjudicationReviews, + RecordedEvidence, + ReviewDecision, + finalize_recorded_results, +) +from benchmarks.agent_cli_comparison import ( + SCHEMA_VERSION, + AgentCliArm, + AgentCliManifest, + BenchmarkTask, + RecordedResults, +) +from benchmarks.agent_cli_recorder import ( + AgentCliRecorderConfig, + AgentCliTrialRecorder, + CommandRunnerPort, + GitWorktreeManager, + LocalCommandRunner, + WorktreeManagerPort, +) + +DEFAULT_BENCHMARK_ID = "agent-cli-local-rehearsal" +DEFAULT_TASK_ID = "local-rehearsal" +DEFAULT_GOAL = "Exercise the recorded benchmark pipeline without external agents." + + +@dataclass(frozen=True) +class LocalRehearsalArtifacts: + manifest: AgentCliManifest + config: AgentCliRecorderConfig + evidence: RecordedEvidence + reviews: AdjudicationReviews + results: RecordedResults + + +def _json_print_script(*payloads: dict[str, object]) -> str: + statements = ["import json"] + statements.extend( + f"print(json.dumps({payload!r}, sort_keys=True))" for payload in payloads + ) + return "; ".join(statements) + + +def build_local_rehearsal_contract( + *, + workspace_revision: str, + python_executable: str = sys.executable, + benchmark_id: str = DEFAULT_BENCHMARK_ID, + task_id: str = DEFAULT_TASK_ID, + goal: str = DEFAULT_GOAL, +) -> tuple[AgentCliManifest, AgentCliRecorderConfig]: + """Build an internal-only three-arm contract whose maximum cost is zero.""" + manifest = AgentCliManifest( + schema_version=SCHEMA_VERSION, + benchmark_id=benchmark_id, + task=BenchmarkTask( + id=task_id, + goal=goal, + workspace_revision=workspace_revision, + checks=("local_process",), + handoff_assertions=("receipt_contract",), + ), + arms=tuple(AgentCliArm), + repetitions=1, + ) + codex = _json_print_script( + { + "type": "turn.completed", + "usage": {"input_tokens": 0, "output_tokens": 0}, + } + ) + claude = _json_print_script( + {"type": "system", "subtype": "init", "model": "local-rehearsal"}, + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "usage": {"input_tokens": 0, "output_tokens": 0}, + "total_cost_usd": 0.0, + }, + ) + morphic = _json_print_script( + { + "type": "morphic_benchmark_receipt", + "success": True, + "model": "morphic-control[local-rehearsal]", + "usage": {"input_tokens": 0, "output_tokens": 0}, + "cost_usd": 0.0, + } + ) + success_command = (python_executable, "-c", "raise SystemExit(0)") + config = AgentCliRecorderConfig( + schema_version=SCHEMA_VERSION, + benchmark_id=benchmark_id, + arm_commands={ + AgentCliArm.CODEX_CLI: (python_executable, "-c", codex), + AgentCliArm.CLAUDE_CODE: (python_executable, "-c", claude), + AgentCliArm.MORPHIC_CONTROL: (python_executable, "-c", morphic), + }, + check_commands={"local_process": success_command}, + handoff_commands={"receipt_contract": success_command}, + estimated_cost_usd_per_trial={arm: 0.0 for arm in AgentCliArm}, + model_hints={AgentCliArm.CODEX_CLI: "local-rehearsal"}, + timeout_seconds=10.0, + ) + return manifest, config + + +async def run_local_rehearsal( + *, + source_root: Path, + worktree_root: Path, + workspace_revision: str, + python_executable: str = sys.executable, + worktree_manager: WorktreeManagerPort | None = None, + command_runner: CommandRunnerPort | None = None, +) -> LocalRehearsalArtifacts: + """Exercise recorder, receipts, review join, and finalizer without external agents.""" + manifest, config = build_local_rehearsal_contract( + workspace_revision=workspace_revision, + python_executable=python_executable, + ) + recorded = await AgentCliTrialRecorder( + worktree_manager=worktree_manager or GitWorktreeManager(), + command_runner=command_runner or LocalCommandRunner(), + ).record( + manifest=manifest, + config=config, + source_root=source_root, + worktree_root=worktree_root, + acknowledged_paid=True, + cost_cap_usd=0.0, + ) + evidence = RecordedEvidence.model_validate(recorded.to_dict()) + decisions = tuple( + ReviewDecision( + arm=trial.arm, + trial=trial.trial, + agent_argv_sha256=trial.agent.argv_sha256, + accepted_patch=False, + human_interventions=0, + recovery_attempted=False, + recovery_succeeded=False, + reviewer_id="local-rehearsal:no-human-review", + review_artifact_sha256=hashlib.sha256( + ( + f"{manifest.benchmark_id}:{trial.arm.value}:{trial.trial}:" + f"{trial.agent.argv_sha256}:local-rehearsal" + ).encode() + ).hexdigest(), + ) + for trial in evidence.trials + ) + reviews = AdjudicationReviews( + schema_version=SCHEMA_VERSION, + benchmark_id=manifest.benchmark_id, + task_id=manifest.task.id, + workspace_revision=manifest.task.workspace_revision, + decisions=decisions, + ) + return LocalRehearsalArtifacts( + manifest=manifest, + config=config, + evidence=evidence, + reviews=reviews, + results=finalize_recorded_results(manifest, evidence, reviews), + ) + + +def _model_json(model: BaseModel) -> str: + return json.dumps(model.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + "\n" + + +def publish_local_rehearsal( + output_dir: Path, + artifacts: LocalRehearsalArtifacts, +) -> None: + """Publish a complete bundle into a newly-created directory only.""" + output_dir.mkdir(mode=0o700, parents=False, exist_ok=False) + try: + payloads = { + "manifest.json": _model_json(artifacts.manifest), + "recorder-config.json": _model_json(artifacts.config), + "evidence.json": _model_json(artifacts.evidence), + "reviews.json": _model_json(artifacts.reviews), + "results.json": _model_json(artifacts.results), + } + for name, payload in payloads.items(): + (output_dir / name).write_text(payload, encoding="utf-8") + except Exception: + shutil.rmtree(output_dir) + raise + + +async def resolve_git_revision( + source_root: Path, + revision: str, + *, + command_runner: CommandRunnerPort | None = None, +) -> str: + """Resolve a revision to a full immutable commit hash without a shell.""" + runner = command_runner or LocalCommandRunner() + capture = await runner.run( + argv=( + "git", + "-C", + str(source_root.resolve()), + "rev-parse", + "--verify", + f"{revision}^{{commit}}", + ), + cwd=source_root.resolve(), + timeout_seconds=30.0, + ) + resolved = capture.stdout.strip() + if capture.exit_code != 0 or len(resolved) != 40: + raise ValueError(f"cannot resolve Git revision {revision!r}") + return resolved + + +if __name__ == "__main__": + raise SystemExit("Use `morphic benchmark agent-cli-rehearse`.") diff --git a/benchmarks/agent_cli_review_policy.py b/benchmarks/agent_cli_review_policy.py new file mode 100644 index 0000000..324fe57 --- /dev/null +++ b/benchmarks/agent_cli_review_policy.py @@ -0,0 +1,126 @@ +"""Structural reviewer separation policy for recorded benchmark campaigns.""" + +from __future__ import annotations + +import hashlib +import json + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_adjudication import AdjudicationReviews +from benchmarks.agent_cli_comparison import SCHEMA_VERSION + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if identifier != identifier.strip() or not identifier: + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +class ReviewerPolicyDeclaration(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + operator_id: str = Field(min_length=1, max_length=200) + reviewer_ids: tuple[str, ...] = Field(min_length=1) + minimum_distinct_reviewers: int = Field(ge=1) + + @model_validator(mode="after") + def validate_contract(self) -> ReviewerPolicyDeclaration: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.operator_id, label="operator_id") + for reviewer_id in self.reviewer_ids: + _validate_identifier(reviewer_id, label="reviewer_id") + if len(set(self.reviewer_ids)) != len(self.reviewer_ids): + raise ValueError("reviewer_ids must be unique") + if self.operator_id in self.reviewer_ids: + raise ValueError("operator must not be an allowed reviewer") + if self.minimum_distinct_reviewers > len(self.reviewer_ids): + raise ValueError("minimum distinct reviewers exceeds allowed reviewers") + return self + + +class ReviewerPolicy(_FrozenModel): + schema_version: int + benchmark_id: str = Field(min_length=1) + operator_id: str = Field(min_length=1, max_length=200) + reviewer_ids: tuple[str, ...] = Field(min_length=1) + minimum_distinct_reviewers: int = Field(ge=1) + policy_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_fingerprint(self) -> ReviewerPolicy: + declaration = ReviewerPolicyDeclaration( + schema_version=self.schema_version, + benchmark_id=self.benchmark_id, + operator_id=self.operator_id, + reviewer_ids=self.reviewer_ids, + minimum_distinct_reviewers=self.minimum_distinct_reviewers, + ) + if tuple(sorted(declaration.reviewer_ids)) != self.reviewer_ids: + raise ValueError("reviewer_ids must be sorted") + expected = _policy_sha256(declaration) + if self.policy_sha256 != expected: + raise ValueError("review policy fingerprint does not match policy") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _policy_sha256(declaration: ReviewerPolicyDeclaration) -> str: + payload = declaration.model_dump(mode="json") + payload["reviewer_ids"] = sorted(payload["reviewer_ids"]) + body = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(body.encode()).hexdigest() + + +def build_reviewer_policy(declaration: ReviewerPolicyDeclaration) -> ReviewerPolicy: + """Normalize a reviewer declaration into a self-fingerprinted policy.""" + return ReviewerPolicy( + schema_version=declaration.schema_version, + benchmark_id=declaration.benchmark_id, + operator_id=declaration.operator_id, + reviewer_ids=tuple(sorted(declaration.reviewer_ids)), + minimum_distinct_reviewers=declaration.minimum_distinct_reviewers, + policy_sha256=_policy_sha256(declaration), + ) + + +def validate_reviewer_separation( + policy: ReviewerPolicy, + reviews: AdjudicationReviews, +) -> None: + """Validate declared structural separation; this is not identity authentication.""" + if reviews.benchmark_id != policy.benchmark_id: + raise ValueError("review policy benchmark_id does not match reviews") + if reviews.review_policy_sha256 != policy.policy_sha256: + raise ValueError("review policy fingerprint does not match reviews") + reviewer_ids = [decision.reviewer_id for decision in reviews.decisions] + if policy.operator_id in reviewer_ids: + raise ValueError("operator must not review campaign decisions") + unauthorized = sorted(set(reviewer_ids) - set(policy.reviewer_ids)) + if unauthorized: + raise ValueError(f"reviewer id is not allowed: {', '.join(unauthorized)}") + if len(set(reviewer_ids)) < policy.minimum_distinct_reviewers: + raise ValueError("distinct reviewer count is below policy minimum") + + +def validate_reviewer_policy_capacity( + policy: ReviewerPolicy, + *, + decision_count: int, +) -> None: + """Reject policies that cannot be satisfied by the campaign decision matrix.""" + if policy.minimum_distinct_reviewers > decision_count: + raise ValueError("minimum distinct reviewers exceeds campaign decision count") diff --git a/benchmarks/agent_cli_transparency.py b/benchmarks/agent_cli_transparency.py new file mode 100644 index 0000000..f54c847 --- /dev/null +++ b/benchmarks/agent_cli_transparency.py @@ -0,0 +1,813 @@ +"""Offline authority-root rotation and append-only benchmark transparency.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from typing import Literal + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_authority import BenchmarkAuthority +from benchmarks.agent_cli_comparison import SCHEMA_VERSION + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" +TransparencyArtifactKind = Literal[ + "authority_root_ledger", + "reviewer_enrollments", + "campaign_envelope", +] + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode()).hexdigest() + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +def _public_key(authority: BenchmarkAuthority) -> Ed25519PublicKey: + return Ed25519PublicKey.from_public_bytes( + _decode_base64( + authority.public_key_base64, + label="authority public_key_base64", + length=32, + ) + ) + + +class AuthorityRotationStatement(_FrozenModel): + schema_version: int + generation: int = Field(ge=2) + predecessor_authority_sha256: str = Field(pattern=_SHA256_PATTERN) + successor_authority_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> AuthorityRotationStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.predecessor_authority_sha256 == self.successor_authority_sha256: + raise ValueError("authority rotation must change the root authority") + return self + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class AuthorityRotationSigningRequest(_FrozenModel): + statement: AuthorityRotationStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_payload(self) -> AuthorityRotationSigningRequest: + expected = self.statement.signing_bytes() + decoded = _decode_base64( + self.signing_payload_base64, + label="authority rotation signing payload", + length=len(expected), + ) + if decoded != expected: + raise ValueError("authority rotation payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class AuthorityRotationCertificate(_FrozenModel): + statement: AuthorityRotationStatement + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature_encoding(self) -> AuthorityRotationCertificate: + _decode_base64( + self.signature_base64, + label="authority rotation signature", + length=64, + ) + return self + + +class AuthorityRootGeneration(_FrozenModel): + schema_version: int + generation: int = Field(ge=1) + authority: BenchmarkAuthority + rotation: AuthorityRotationCertificate | None = None + + @model_validator(mode="after") + def validate_generation(self) -> AuthorityRootGeneration: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if (self.generation == 1) != (self.rotation is None): + raise ValueError("only the genesis authority may omit a rotation certificate") + return self + + +class AuthorityRootLedgerStatement(_FrozenModel): + schema_version: int + generations: tuple[AuthorityRootGeneration, ...] = Field(min_length=1) + revoked_authority_sha256: tuple[str, ...] = () + active_generation: int = Field(ge=1) + ledger_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> AuthorityRootLedgerStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.active_generation != self.generations[-1].generation: + raise ValueError("active_generation must identify the latest root generation") + revoked = self.revoked_authority_sha256 + if tuple(sorted(set(revoked))) != revoked: + raise ValueError("revoked authority fingerprints must be sorted and unique") + if any( + len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + for value in revoked + ): + raise ValueError("revoked authority fingerprints must be SHA-256 hex") + if self.ledger_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("authority root ledger fingerprint does not match statement") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"ledger_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class AuthorityRootLedgerSigningRequest(_FrozenModel): + statement: AuthorityRootLedgerStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_payload(self) -> AuthorityRootLedgerSigningRequest: + expected = self.statement.signing_bytes() + try: + decoded = base64.b64decode(self.signing_payload_base64, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError("authority root ledger payload must be canonical base64") from exc + if decoded != expected or base64.b64encode(decoded).decode() != self.signing_payload_base64: + raise ValueError("authority root ledger payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class SignedAuthorityRootLedger(_FrozenModel): + statement: AuthorityRootLedgerStatement + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature_encoding(self) -> SignedAuthorityRootLedger: + _decode_base64( + self.signature_base64, + label="authority root ledger signature", + length=64, + ) + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_authority_rotation_request( + *, + generation: int, + predecessor: BenchmarkAuthority, + successor: BenchmarkAuthority, +) -> AuthorityRotationSigningRequest: + """Create a predecessor-key signing payload for one root rotation.""" + statement = AuthorityRotationStatement( + schema_version=SCHEMA_VERSION, + generation=generation, + predecessor_authority_sha256=predecessor.authority_sha256, + successor_authority_sha256=successor.authority_sha256, + ) + return AuthorityRotationSigningRequest( + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def _validate_generation_chain( + generations: tuple[AuthorityRootGeneration, ...], +) -> None: + if not generations: + raise ValueError("authority root ledger requires at least one generation") + if generations[0].generation != 1 or generations[0].rotation is not None: + raise ValueError("authority root ledger must begin with an unsigned genesis") + fingerprints: set[str] = set() + for index, generation in enumerate(generations): + if generation.generation != index + 1: + raise ValueError("authority root generations must be contiguous and ordered") + fingerprint = generation.authority.authority_sha256 + if fingerprint in fingerprints: + raise ValueError("authority root generations must not reuse an authority") + fingerprints.add(fingerprint) + if index == 0: + continue + predecessor = generations[index - 1].authority + rotation = generation.rotation + if rotation is None: + raise ValueError("rotated authority generation is missing its certificate") + expected = AuthorityRotationStatement( + schema_version=SCHEMA_VERSION, + generation=generation.generation, + predecessor_authority_sha256=predecessor.authority_sha256, + successor_authority_sha256=fingerprint, + ) + if rotation.statement != expected: + raise ValueError("authority rotation statement does not match generation chain") + try: + _public_key(predecessor).verify( + _decode_base64( + rotation.signature_base64, + label="authority rotation signature", + length=64, + ), + rotation.statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError("authority rotation signature is invalid") from exc + + +def build_authority_root_ledger_request( + generations: tuple[AuthorityRootGeneration, ...], + *, + revoked_authority_sha256: tuple[str, ...] = (), +) -> AuthorityRootLedgerSigningRequest: + """Create an active-root signing payload after verifying the full rotation chain.""" + _validate_generation_chain(generations) + known = {generation.authority.authority_sha256 for generation in generations} + revoked = tuple(sorted(set(revoked_authority_sha256))) + unknown = set(revoked) - known + if unknown: + raise ValueError("revoked authority is not present in the root ledger") + active = generations[-1].authority.authority_sha256 + if active in revoked: + raise ValueError("active authority is revoked") + payload = { + "schema_version": SCHEMA_VERSION, + "generations": [item.model_dump(mode="json") for item in generations], + "revoked_authority_sha256": list(revoked), + "active_generation": generations[-1].generation, + } + statement = AuthorityRootLedgerStatement( + **payload, + ledger_sha256=_canonical_sha256(payload), + ) + return AuthorityRootLedgerSigningRequest( + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def verify_authority_root_ledger( + ledger: SignedAuthorityRootLedger, +) -> BenchmarkAuthority: + """Verify rotations, revocations, and the latest authority's ledger signature.""" + statement = AuthorityRootLedgerStatement.model_validate( + ledger.statement.model_dump(mode="json") + ) + if statement.ledger_sha256 != _canonical_sha256(statement._binding_payload()): + raise ValueError("authority root ledger fingerprint does not match statement") + _validate_generation_chain(statement.generations) + known = { + generation.authority.authority_sha256 for generation in statement.generations + } + if set(statement.revoked_authority_sha256) - known: + raise ValueError("revoked authority is not present in the root ledger") + active = statement.generations[-1].authority + if active.authority_sha256 in statement.revoked_authority_sha256: + raise ValueError("active authority is revoked") + try: + _public_key(active).verify( + _decode_base64( + ledger.signature_base64, + label="authority root ledger signature", + length=64, + ), + statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError("authority root ledger signature is invalid") from exc + return active + + +class TransparencyLogEntry(_FrozenModel): + sequence: int = Field(ge=0) + kind: TransparencyArtifactKind + artifact_sha256: str = Field(pattern=_SHA256_PATTERN) + + def leaf_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +def _leaf_hash(entry: TransparencyLogEntry) -> bytes: + return hashlib.sha256(b"\x00" + entry.leaf_bytes()).digest() + + +def _node_hash(left: bytes, right: bytes) -> bytes: + return hashlib.sha256(b"\x01" + left + right).digest() + + +def _largest_power_of_two_less_than(value: int) -> int: + return 1 << ((value - 1).bit_length() - 1) + + +def _merkle_root(entries: tuple[TransparencyLogEntry, ...]) -> bytes: + if not entries: + return hashlib.sha256(b"").digest() + if len(entries) == 1: + return _leaf_hash(entries[0]) + split = _largest_power_of_two_less_than(len(entries)) + return _node_hash(_merkle_root(entries[:split]), _merkle_root(entries[split:])) + + +class TransparencyLog(_FrozenModel): + schema_version: int + log_id: str = Field(min_length=1, max_length=200) + entries: tuple[TransparencyLogEntry, ...] + tree_size: int = Field(ge=0) + root_sha256: str = Field(pattern=_SHA256_PATTERN) + log_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_log(self) -> TransparencyLog: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if not self.log_id or self.log_id != self.log_id.strip(): + raise ValueError("log_id must be non-blank without surrounding whitespace") + if tuple(entry.sequence for entry in self.entries) != tuple(range(len(self.entries))): + raise ValueError("transparency log entries must have contiguous sequence numbers") + if self.tree_size != len(self.entries): + raise ValueError("transparency tree_size does not match entries") + if self.root_sha256 != _merkle_root(self.entries).hex(): + raise ValueError("transparency root does not match entries") + if self.log_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("transparency log fingerprint does not match entries") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"log_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_transparency_log( + log_id: str, + entries: tuple[TransparencyLogEntry, ...], +) -> TransparencyLog: + """Build a complete deterministic RFC 6962-style Merkle log artifact.""" + normalized = tuple( + entry.model_copy(update={"sequence": index}) + for index, entry in enumerate(entries) + ) + payload = { + "schema_version": SCHEMA_VERSION, + "log_id": log_id, + "entries": [entry.model_dump(mode="json") for entry in normalized], + "tree_size": len(normalized), + "root_sha256": _merkle_root(normalized).hex(), + } + return TransparencyLog(**payload, log_sha256=_canonical_sha256(payload)) + + +def extend_transparency_log( + previous: TransparencyLog, + entries: tuple[TransparencyLogEntry, ...], +) -> TransparencyLog: + """Append entries to a complete log without mutating its existing prefix.""" + return build_transparency_log(previous.log_id, previous.entries + entries) + + +def verify_complete_log_extension( + previous: TransparencyLog, + current: TransparencyLog, +) -> None: + """Verify append-only growth when both complete log artifacts are available.""" + previous = TransparencyLog.model_validate(previous.model_dump(mode="json")) + current = TransparencyLog.model_validate(current.model_dump(mode="json")) + if previous.log_id != current.log_id: + raise ValueError("transparency log_id changed") + if current.tree_size < previous.tree_size: + raise ValueError("transparency log is not an append-only extension") + if current.entries[: previous.tree_size] != previous.entries: + raise ValueError("transparency log is not an append-only extension") + + +class TransparencyTreeHeadStatement(_FrozenModel): + schema_version: int + log_id: str = Field(min_length=1, max_length=200) + tree_size: int = Field(ge=0) + root_sha256: str = Field(pattern=_SHA256_PATTERN) + authority_root_ledger_sha256: str = Field(pattern=_SHA256_PATTERN) + tree_head_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> TransparencyTreeHeadStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.tree_head_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("transparency tree head fingerprint does not match statement") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"tree_head_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class TransparencyTreeHeadSigningRequest(_FrozenModel): + statement: TransparencyTreeHeadStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_payload(self) -> TransparencyTreeHeadSigningRequest: + expected = self.statement.signing_bytes() + try: + decoded = base64.b64decode(self.signing_payload_base64, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError("transparency tree head payload must be canonical base64") from exc + if decoded != expected or base64.b64encode(decoded).decode() != self.signing_payload_base64: + raise ValueError("transparency tree head payload does not match statement") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class SignedTransparencyTreeHead(_FrozenModel): + statement: TransparencyTreeHeadStatement + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature_encoding(self) -> SignedTransparencyTreeHead: + _decode_base64( + self.signature_base64, + label="transparency tree head signature", + length=64, + ) + return self + + +class TransparencyInclusionProof(_FrozenModel): + schema_version: int + entry: TransparencyLogEntry + leaf_index: int = Field(ge=0) + tree_size: int = Field(ge=1) + audit_path_sha256: tuple[str, ...] + tree_head: SignedTransparencyTreeHead + + @model_validator(mode="after") + def validate_proof(self) -> TransparencyInclusionProof: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.leaf_index >= self.tree_size: + raise ValueError("transparency leaf_index is outside tree_size") + if self.entry.sequence != self.leaf_index: + raise ValueError("transparency entry sequence does not match leaf_index") + if self.tree_head.statement.tree_size != self.tree_size: + raise ValueError("transparency proof tree_size does not match tree head") + for digest in self.audit_path_sha256: + if len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError("transparency audit path must contain SHA-256 hex") + return self + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class TransparencyConsistencyProof(_FrozenModel): + schema_version: int + previous_tree_head: SignedTransparencyTreeHead + current_tree_head: SignedTransparencyTreeHead + audit_path_sha256: tuple[str, ...] + consistency_proof_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_proof(self) -> TransparencyConsistencyProof: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + previous = self.previous_tree_head.statement + current = self.current_tree_head.statement + if previous.log_id != current.log_id: + raise ValueError("consistency proof tree heads must use the same log_id") + if previous.tree_size < 1 or previous.tree_size >= current.tree_size: + raise ValueError("consistency proof requires 0 < previous size < current size") + if ( + previous.authority_root_ledger_sha256 + != current.authority_root_ledger_sha256 + ): + raise ValueError("consistency proof tree heads use different root ledgers") + for digest in self.audit_path_sha256: + if len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError("consistency audit path must contain SHA-256 hex") + if self.consistency_proof_sha256 != _canonical_sha256( + self._binding_payload() + ): + raise ValueError("consistency proof fingerprint does not match tree heads") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"consistency_proof_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def build_transparency_tree_head_request( + log: TransparencyLog, + ledger: SignedAuthorityRootLedger, +) -> TransparencyTreeHeadSigningRequest: + """Create an active-root signing payload for one Merkle tree head.""" + verify_authority_root_ledger(ledger) + payload = { + "schema_version": SCHEMA_VERSION, + "log_id": log.log_id, + "tree_size": log.tree_size, + "root_sha256": log.root_sha256, + "authority_root_ledger_sha256": ledger.statement.ledger_sha256, + } + statement = TransparencyTreeHeadStatement( + **payload, + tree_head_sha256=_canonical_sha256(payload), + ) + return TransparencyTreeHeadSigningRequest( + statement=statement, + signing_payload_base64=base64.b64encode(statement.signing_bytes()).decode(), + ) + + +def verify_signed_transparency_tree_head( + tree_head: SignedTransparencyTreeHead, + ledger: SignedAuthorityRootLedger, +) -> TransparencyTreeHeadStatement: + """Verify one tree head against the active authority-root ledger.""" + active = verify_authority_root_ledger(ledger) + statement = TransparencyTreeHeadStatement.model_validate( + tree_head.statement.model_dump(mode="json") + ) + if statement.authority_root_ledger_sha256 != ledger.statement.ledger_sha256: + raise ValueError("transparency tree head does not match authority root ledger") + try: + _public_key(active).verify( + _decode_base64( + tree_head.signature_base64, + label="transparency tree head signature", + length=64, + ), + statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError("transparency tree head signature is invalid") from exc + return statement + + +def _consistency_path( + entries: tuple[TransparencyLogEntry, ...], + previous_size: int, + *, + complete_subtree: bool, +) -> tuple[bytes, ...]: + if previous_size == len(entries): + return () if complete_subtree else (_merkle_root(entries),) + split = _largest_power_of_two_less_than(len(entries)) + if previous_size <= split: + return _consistency_path( + entries[:split], + previous_size, + complete_subtree=complete_subtree, + ) + (_merkle_root(entries[split:]),) + return _consistency_path( + entries[split:], + previous_size - split, + complete_subtree=False, + ) + (_merkle_root(entries[:split]),) + + +def build_transparency_consistency_proof( + current_log: TransparencyLog, + *, + previous_tree_head: SignedTransparencyTreeHead, + current_tree_head: SignedTransparencyTreeHead, + authority_root_ledger: SignedAuthorityRootLedger, +) -> TransparencyConsistencyProof: + """Build the RFC 6962 minimal consistency proof from one complete current log.""" + current_log = TransparencyLog.model_validate(current_log.model_dump(mode="json")) + previous = verify_signed_transparency_tree_head( + previous_tree_head, + authority_root_ledger, + ) + current = verify_signed_transparency_tree_head( + current_tree_head, + authority_root_ledger, + ) + if (current.log_id, current.tree_size, current.root_sha256) != ( + current_log.log_id, + current_log.tree_size, + current_log.root_sha256, + ): + raise ValueError("current consistency tree head does not match complete log") + if previous.log_id != current.log_id: + raise ValueError("consistency proof tree heads must use the same log_id") + if previous.tree_size < 1 or previous.tree_size >= current.tree_size: + raise ValueError("consistency proof requires 0 < previous size < current size") + if _merkle_root(current_log.entries[: previous.tree_size]).hex() != ( + previous.root_sha256 + ): + raise ValueError("previous tree head is not a prefix root of current log") + audit_path = tuple( + item.hex() + for item in _consistency_path( + current_log.entries, + previous.tree_size, + complete_subtree=True, + ) + ) + payload = { + "schema_version": SCHEMA_VERSION, + "previous_tree_head": previous_tree_head.model_dump(mode="json"), + "current_tree_head": current_tree_head.model_dump(mode="json"), + "audit_path_sha256": list(audit_path), + } + return TransparencyConsistencyProof( + **payload, + consistency_proof_sha256=_canonical_sha256(payload), + ) + + +def verify_transparency_consistency_proof( + proof: TransparencyConsistencyProof, + ledger: SignedAuthorityRootLedger, +) -> None: + """Verify append-only growth using only two signed roots and a compact path.""" + proof = TransparencyConsistencyProof.model_validate(proof.model_dump(mode="json")) + previous = verify_signed_transparency_tree_head(proof.previous_tree_head, ledger) + current = verify_signed_transparency_tree_head(proof.current_tree_head, ledger) + old_size = previous.tree_size + new_size = current.tree_size + old_index = old_size - 1 + new_index = new_size - 1 + while old_index & 1: + old_index >>= 1 + new_index >>= 1 + path = tuple(bytes.fromhex(item) for item in proof.audit_path_sha256) + position = 0 + if old_index == 0: + old_root = bytes.fromhex(previous.root_sha256) + new_root = old_root + else: + if not path: + raise ValueError("consistency proof audit path is incomplete") + old_root = path[0] + new_root = path[0] + position = 1 + while position < len(path): + if new_index == 0: + raise ValueError("consistency proof audit path has extra nodes") + node = path[position] + if old_index & 1 or old_index == new_index: + old_root = _node_hash(node, old_root) + new_root = _node_hash(node, new_root) + while old_index and not old_index & 1: + old_index >>= 1 + new_index >>= 1 + else: + new_root = _node_hash(new_root, node) + old_index >>= 1 + new_index >>= 1 + position += 1 + if ( + old_root.hex() != previous.root_sha256 + or new_root.hex() != current.root_sha256 + ): + raise ValueError("consistency proof roots do not match signed tree heads") + + +def _audit_path( + entries: tuple[TransparencyLogEntry, ...], + leaf_index: int, +) -> tuple[bytes, ...]: + if len(entries) == 1: + return () + split = _largest_power_of_two_less_than(len(entries)) + if leaf_index < split: + return _audit_path(entries[:split], leaf_index) + ( + _merkle_root(entries[split:]), + ) + return _audit_path(entries[split:], leaf_index - split) + ( + _merkle_root(entries[:split]), + ) + + +def build_transparency_inclusion_proof( + log: TransparencyLog, + *, + leaf_index: int, + tree_head: SignedTransparencyTreeHead, +) -> TransparencyInclusionProof: + """Build an inclusion proof against an already-signed matching tree head.""" + if leaf_index < 0 or leaf_index >= log.tree_size: + raise ValueError("transparency leaf_index is outside tree_size") + statement = tree_head.statement + if ( + statement.log_id, + statement.tree_size, + statement.root_sha256, + ) != (log.log_id, log.tree_size, log.root_sha256): + raise ValueError("signed transparency tree head does not match log") + return TransparencyInclusionProof( + schema_version=SCHEMA_VERSION, + entry=log.entries[leaf_index], + leaf_index=leaf_index, + tree_size=log.tree_size, + audit_path_sha256=tuple(item.hex() for item in _audit_path(log.entries, leaf_index)), + tree_head=tree_head, + ) + + +def _root_from_audit_path( + leaf_hash: bytes, + *, + leaf_index: int, + tree_size: int, + audit_path: tuple[bytes, ...], +) -> bytes: + position = 0 + + def rebuild(current: bytes, index: int, size: int) -> bytes: + nonlocal position + if size == 1: + return current + split = _largest_power_of_two_less_than(size) + if index < split: + left = rebuild(current, index, split) + if position >= len(audit_path): + raise ValueError("transparency inclusion proof audit path is incomplete") + right = audit_path[position] + position += 1 + return _node_hash(left, right) + right = rebuild(current, index - split, size - split) + if position >= len(audit_path): + raise ValueError("transparency inclusion proof audit path is incomplete") + left = audit_path[position] + position += 1 + return _node_hash(left, right) + + root = rebuild(leaf_hash, leaf_index, tree_size) + if position != len(audit_path): + raise ValueError("transparency inclusion proof audit path has extra nodes") + return root + + +def verify_transparency_inclusion_proof( + proof: TransparencyInclusionProof, + ledger: SignedAuthorityRootLedger, + *, + expected_kind: TransparencyArtifactKind, + expected_artifact_sha256: str, +) -> None: + """Verify the artifact leaf, signed tree head, and authority-root ledger.""" + proof = TransparencyInclusionProof.model_validate(proof.model_dump(mode="json")) + statement = verify_signed_transparency_tree_head(proof.tree_head, ledger) + if (proof.entry.kind, proof.entry.artifact_sha256) != ( + expected_kind, + expected_artifact_sha256, + ): + raise ValueError("transparency inclusion proof does not match expected artifact") + root = _root_from_audit_path( + _leaf_hash(proof.entry), + leaf_index=proof.leaf_index, + tree_size=proof.tree_size, + audit_path=tuple(bytes.fromhex(item) for item in proof.audit_path_sha256), + ) + if root.hex() != statement.root_sha256: + raise ValueError("transparency inclusion proof root does not match tree head") diff --git a/benchmarks/agent_cli_witness.py b/benchmarks/agent_cli_witness.py new file mode 100644 index 0000000..f8d39d1 --- /dev/null +++ b/benchmarks/agent_cli_witness.py @@ -0,0 +1,472 @@ +"""Offline quorum witnesses for benchmark transparency checkpoints.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from typing import Literal + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_transparency import ( + SignedAuthorityRootLedger, + TransparencyConsistencyProof, + verify_transparency_consistency_proof, +) + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(allow_inf_nan=False, extra="forbid", frozen=True) + + +def _canonical_json(payload: object) -> str: + return json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256(_canonical_json(payload).encode()).hexdigest() + + +def _validate_identifier(identifier: str, *, label: str) -> None: + if not identifier or identifier != identifier.strip(): + raise ValueError(f"{label} must be non-blank without surrounding whitespace") + + +def _decode_base64(value: str, *, label: str, length: int) -> bytes: + try: + decoded = base64.b64decode(value, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} must be canonical base64") from exc + if len(decoded) != length or base64.b64encode(decoded).decode() != value: + raise ValueError(f"{label} must encode exactly {length} bytes") + return decoded + + +class TransparencyWitnessKeyDeclaration(_FrozenModel): + witness_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + status: Literal["active", "revoked"] = "active" + + @model_validator(mode="after") + def validate_key(self) -> TransparencyWitnessKeyDeclaration: + _validate_identifier(self.witness_id, label="witness_id") + _validate_identifier(self.key_id, label="key_id") + _decode_base64( + self.public_key_base64, + label="witness public_key_base64", + length=32, + ) + return self + + +class TransparencyWitnessTrustDeclaration(_FrozenModel): + schema_version: int + log_id: str = Field(min_length=1, max_length=200) + minimum_distinct_witnesses: int = Field(ge=1) + keys: tuple[TransparencyWitnessKeyDeclaration, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_declaration(self) -> TransparencyWitnessTrustDeclaration: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + _validate_identifier(self.log_id, label="log_id") + identities = [(key.witness_id, key.key_id) for key in self.keys] + if len(identities) != len(set(identities)): + raise ValueError("witness key identities must be unique") + if len({key.key_id for key in self.keys}) != len(self.keys): + raise ValueError("witness key_id values must be globally unique") + return self + + +class TransparencyWitnessKey(_FrozenModel): + witness_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + algorithm: Literal["ed25519"] = "ed25519" + public_key_base64: str = Field(min_length=1) + public_key_sha256: str = Field(pattern=_SHA256_PATTERN) + status: Literal["active", "revoked"] + + @model_validator(mode="after") + def validate_fingerprint(self) -> TransparencyWitnessKey: + declaration = TransparencyWitnessKeyDeclaration( + witness_id=self.witness_id, + key_id=self.key_id, + algorithm=self.algorithm, + public_key_base64=self.public_key_base64, + status=self.status, + ) + public_key = _decode_base64( + declaration.public_key_base64, + label="witness public_key_base64", + length=32, + ) + if self.public_key_sha256 != hashlib.sha256(public_key).hexdigest(): + raise ValueError("witness public key fingerprint does not match key") + return self + + +class TransparencyWitnessTrust(_FrozenModel): + schema_version: int + log_id: str = Field(min_length=1, max_length=200) + minimum_distinct_witnesses: int = Field(ge=1) + keys: tuple[TransparencyWitnessKey, ...] = Field(min_length=1) + witness_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_trust(self) -> TransparencyWitnessTrust: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if tuple(sorted(self.keys, key=_key_identity)) != self.keys: + raise ValueError("witness keys must be sorted") + if self.witness_trust_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("witness trust fingerprint does not match declaration") + _validate_witness_capacity( + self.keys, + minimum_distinct_witnesses=self.minimum_distinct_witnesses, + ) + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"witness_trust_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _key_identity( + key: TransparencyWitnessKey | TransparencyWitnessKeyDeclaration, +) -> tuple[str, str]: + return key.witness_id, key.key_id + + +def _validate_witness_capacity( + keys: tuple[TransparencyWitnessKey, ...], + *, + minimum_distinct_witnesses: int, +) -> None: + active_witnesses = {key.witness_id for key in keys if key.status == "active"} + declared_witnesses = {key.witness_id for key in keys} + missing = sorted(declared_witnesses - active_witnesses) + if missing: + raise ValueError(f"witness has no active key: {', '.join(missing)}") + if minimum_distinct_witnesses > len(active_witnesses): + raise ValueError("witness quorum exceeds active witness capacity") + if minimum_distinct_witnesses <= len(active_witnesses) // 2: + raise ValueError("witness quorum must be a strict majority") + + +def build_transparency_witness_trust( + declaration: TransparencyWitnessTrustDeclaration, +) -> TransparencyWitnessTrust: + """Normalize witness public keys and require an intersecting active quorum.""" + keys = tuple( + sorted( + ( + TransparencyWitnessKey( + witness_id=key.witness_id, + key_id=key.key_id, + algorithm=key.algorithm, + public_key_base64=key.public_key_base64, + public_key_sha256=hashlib.sha256( + _decode_base64( + key.public_key_base64, + label="witness public_key_base64", + length=32, + ) + ).hexdigest(), + status=key.status, + ) + for key in declaration.keys + ), + key=_key_identity, + ) + ) + _validate_witness_capacity( + keys, + minimum_distinct_witnesses=declaration.minimum_distinct_witnesses, + ) + payload = { + "schema_version": declaration.schema_version, + "log_id": declaration.log_id, + "minimum_distinct_witnesses": declaration.minimum_distinct_witnesses, + "keys": [key.model_dump(mode="json") for key in keys], + } + return TransparencyWitnessTrust( + **payload, + witness_trust_sha256=_canonical_sha256(payload), + ) + + +class WitnessCheckpointStatement(_FrozenModel): + schema_version: int + log_id: str = Field(min_length=1, max_length=200) + previous_tree_size: int = Field(ge=1) + previous_root_sha256: str = Field(pattern=_SHA256_PATTERN) + previous_tree_head_sha256: str = Field(pattern=_SHA256_PATTERN) + current_tree_size: int = Field(ge=2) + current_root_sha256: str = Field(pattern=_SHA256_PATTERN) + current_tree_head_sha256: str = Field(pattern=_SHA256_PATTERN) + authority_root_ledger_sha256: str = Field(pattern=_SHA256_PATTERN) + consistency_proof_sha256: str = Field(pattern=_SHA256_PATTERN) + witness_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + checkpoint_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_statement(self) -> WitnessCheckpointStatement: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if self.previous_tree_size >= self.current_tree_size: + raise ValueError("witness checkpoint requires increasing tree sizes") + if self.checkpoint_sha256 != _canonical_sha256(self._binding_payload()): + raise ValueError("witness checkpoint fingerprint does not match statement") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"checkpoint_sha256"}) + + def signing_bytes(self) -> bytes: + return _canonical_json(self.model_dump(mode="json")).encode() + + +class WitnessCheckpointSigningRequest(_FrozenModel): + witness_id: str = Field(min_length=1, max_length=200) + eligible_key_ids: tuple[str, ...] = Field(min_length=1) + statement: WitnessCheckpointStatement + signing_payload_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_request(self) -> WitnessCheckpointSigningRequest: + _validate_identifier(self.witness_id, label="witness_id") + if tuple(sorted(set(self.eligible_key_ids))) != self.eligible_key_ids: + raise ValueError("eligible witness key IDs must be sorted and unique") + expected = self.statement.signing_bytes() + try: + decoded = base64.b64decode(self.signing_payload_base64, validate=True) + except (TypeError, ValueError) as exc: + raise ValueError("witness checkpoint payload must be canonical base64") from exc + if decoded != expected or base64.b64encode(decoded).decode() != ( + self.signing_payload_base64 + ): + raise ValueError("witness checkpoint payload does not match statement") + return self + + +class WitnessCheckpointTemplate(_FrozenModel): + schema_version: int + witness_trust_sha256: str = Field(pattern=_SHA256_PATTERN) + consistency_proof_sha256: str = Field(pattern=_SHA256_PATTERN) + statement: WitnessCheckpointStatement + requests: tuple[WitnessCheckpointSigningRequest, ...] = Field(min_length=1) + signatures_completed: Literal[False] = False + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +class TransparencyWitnessSignature(_FrozenModel): + witness_id: str = Field(min_length=1, max_length=200) + key_id: str = Field(min_length=1, max_length=200) + signature_base64: str = Field(min_length=1) + + @model_validator(mode="after") + def validate_signature(self) -> TransparencyWitnessSignature: + _validate_identifier(self.witness_id, label="witness_id") + _validate_identifier(self.key_id, label="key_id") + _decode_base64( + self.signature_base64, + label="witness checkpoint signature", + length=64, + ) + return self + + +class SignedWitnessCheckpoint(_FrozenModel): + schema_version: int + statement: WitnessCheckpointStatement + signatures: tuple[TransparencyWitnessSignature, ...] = Field(min_length=1) + witness_checkpoint_sha256: str = Field(pattern=_SHA256_PATTERN) + + @model_validator(mode="after") + def validate_bundle(self) -> SignedWitnessCheckpoint: + if self.schema_version != SCHEMA_VERSION: + raise ValueError(f"schema_version must be {SCHEMA_VERSION}") + if tuple(sorted(self.signatures, key=_signature_identity)) != self.signatures: + raise ValueError("witness signatures must be sorted") + witness_ids = [signature.witness_id for signature in self.signatures] + if len(witness_ids) != len(set(witness_ids)): + raise ValueError("witness checkpoint signatures must use distinct witnesses") + if self.witness_checkpoint_sha256 != _canonical_sha256( + self._binding_payload() + ): + raise ValueError("witness checkpoint bundle fingerprint does not match") + return self + + def _binding_payload(self) -> dict[str, object]: + return self.model_dump(mode="json", exclude={"witness_checkpoint_sha256"}) + + def to_json(self) -> str: + return json.dumps(self.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + + +def _signature_identity( + signature: TransparencyWitnessSignature, +) -> tuple[str, str]: + return signature.witness_id, signature.key_id + + +def _checkpoint_statement( + trust: TransparencyWitnessTrust, + proof: TransparencyConsistencyProof, + ledger: SignedAuthorityRootLedger, +) -> WitnessCheckpointStatement: + verify_transparency_consistency_proof(proof, ledger) + previous = proof.previous_tree_head.statement + current = proof.current_tree_head.statement + if trust.log_id != current.log_id: + raise ValueError("witness trust log_id does not match consistency proof") + payload = { + "schema_version": SCHEMA_VERSION, + "log_id": current.log_id, + "previous_tree_size": previous.tree_size, + "previous_root_sha256": previous.root_sha256, + "previous_tree_head_sha256": previous.tree_head_sha256, + "current_tree_size": current.tree_size, + "current_root_sha256": current.root_sha256, + "current_tree_head_sha256": current.tree_head_sha256, + "authority_root_ledger_sha256": ledger.statement.ledger_sha256, + "consistency_proof_sha256": proof.consistency_proof_sha256, + "witness_trust_sha256": trust.witness_trust_sha256, + } + return WitnessCheckpointStatement( + **payload, + checkpoint_sha256=_canonical_sha256(payload), + ) + + +def build_witness_checkpoint_template( + trust: TransparencyWitnessTrust, + proof: TransparencyConsistencyProof, + ledger: SignedAuthorityRootLedger, +) -> WitnessCheckpointTemplate: + """Create one private-key-free checkpoint request per active witness.""" + statement = _checkpoint_statement(trust, proof, ledger) + active_keys: dict[str, list[str]] = {} + for key in trust.keys: + if key.status == "active": + active_keys.setdefault(key.witness_id, []).append(key.key_id) + requests = tuple( + WitnessCheckpointSigningRequest( + witness_id=witness_id, + eligible_key_ids=tuple(sorted(key_ids)), + statement=statement, + signing_payload_base64=base64.b64encode( + statement.signing_bytes() + ).decode(), + ) + for witness_id, key_ids in sorted(active_keys.items()) + ) + return WitnessCheckpointTemplate( + schema_version=SCHEMA_VERSION, + witness_trust_sha256=trust.witness_trust_sha256, + consistency_proof_sha256=proof.consistency_proof_sha256, + statement=statement, + requests=requests, + ) + + +def build_witness_checkpoint_bundle( + trust: TransparencyWitnessTrust, + proof: TransparencyConsistencyProof, + ledger: SignedAuthorityRootLedger, + signatures: tuple[TransparencyWitnessSignature, ...], +) -> SignedWitnessCheckpoint: + """Normalize and verify a quorum of detached witness signatures.""" + statement = _checkpoint_statement(trust, proof, ledger) + ordered = tuple(sorted(signatures, key=_signature_identity)) + payload = { + "schema_version": SCHEMA_VERSION, + "statement": statement.model_dump(mode="json"), + "signatures": [signature.model_dump(mode="json") for signature in ordered], + } + bundle = SignedWitnessCheckpoint( + **payload, + witness_checkpoint_sha256=_canonical_sha256(payload), + ) + verify_witness_checkpoint_bundle(trust, proof, ledger, bundle) + return bundle + + +def verify_witness_checkpoint_bundle( + trust: TransparencyWitnessTrust, + proof: TransparencyConsistencyProof, + ledger: SignedAuthorityRootLedger, + bundle: SignedWitnessCheckpoint, +) -> None: + """Verify exact checkpoint binding and an intersecting witness quorum.""" + trust = TransparencyWitnessTrust.model_validate(trust.model_dump(mode="json")) + bundle = SignedWitnessCheckpoint.model_validate(bundle.model_dump(mode="json")) + expected_statement = _checkpoint_statement(trust, proof, ledger) + if bundle.statement != expected_statement: + raise ValueError("witness checkpoint does not match consistency proof") + if len(bundle.signatures) < trust.minimum_distinct_witnesses: + raise ValueError("witness quorum is incomplete") + active_keys = { + (key.witness_id, key.key_id): key for key in trust.keys if key.status == "active" + } + for signature in bundle.signatures: + key = active_keys.get(_signature_identity(signature)) + if key is None: + raise ValueError("witness signature does not use an active trusted key") + try: + Ed25519PublicKey.from_public_bytes( + _decode_base64( + key.public_key_base64, + label="witness public_key_base64", + length=32, + ) + ).verify( + _decode_base64( + signature.signature_base64, + label="witness checkpoint signature", + length=64, + ), + bundle.statement.signing_bytes(), + ) + except (InvalidSignature, ValueError) as exc: + raise ValueError( + f"witness signature is invalid: {signature.witness_id}" + ) from exc + + +def detect_witness_checkpoint_conflict( + first: SignedWitnessCheckpoint, + second: SignedWitnessCheckpoint, +) -> None: + """Reject two witnessed roots for the same log and tree size.""" + first_statement = SignedWitnessCheckpoint.model_validate( + first.model_dump(mode="json") + ).statement + second_statement = SignedWitnessCheckpoint.model_validate( + second.model_dump(mode="json") + ).statement + if first_statement.log_id != second_statement.log_id: + return + if ( + first_statement.current_tree_size == second_statement.current_tree_size + and first_statement.current_root_sha256 + != second_statement.current_root_sha256 + ): + raise ValueError("split-view checkpoint detected for the same tree size") diff --git a/benchmarks/templates/agent_cli_anchored_reviewer_trust.example.json b/benchmarks/templates/agent_cli_anchored_reviewer_trust.example.json new file mode 100644 index 0000000..8c8e5a2 --- /dev/null +++ b/benchmarks/templates/agent_cli_anchored_reviewer_trust.example.json @@ -0,0 +1,22 @@ +{ + "benchmark_id": "agent-cli-local-rehearsal", + "keys": [ + { + "algorithm": "ed25519", + "key_id": "REPLACE_WITH_REVIEWER_1_KEY_ID", + "public_key_base64": "iojj3XQJ8ZX9UtstPLpdcspnCb8dlBIb83SIAbQPb1w=", + "reviewer_id": "REPLACE_WITH_REVIEWER_1_ID", + "status": "active" + }, + { + "algorithm": "ed25519", + "key_id": "REPLACE_WITH_REVIEWER_2_KEY_ID", + "public_key_base64": "gTl3Dqh9F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5Q=", + "reviewer_id": "REPLACE_WITH_REVIEWER_2_ID", + "status": "active" + } + ], + "review_policy_sha256": "b2b73f5017d25f3dab6b180cf9c8c2d248f9d341a0fde28c7ba392e103df2b09", + "reviewer_authority_sha256": "2d978b005dd7947a66dd80603d14fb44c4a6b865551bffd2203341f3f28b0d7e", + "schema_version": 1 +} diff --git a/benchmarks/templates/agent_cli_checkpoint_peer_trust.example.json b/benchmarks/templates/agent_cli_checkpoint_peer_trust.example.json new file mode 100644 index 0000000..6716a00 --- /dev/null +++ b/benchmarks/templates/agent_cli_checkpoint_peer_trust.example.json @@ -0,0 +1,27 @@ +{ + "keys": [ + { + "algorithm": "ed25519", + "key_id": "registry-peer-a-key-1", + "peer_id": "registry-peer-a", + "public_key_base64": "QwRr/kCSs+lJlOraFdzCDYqqB7ZY/TlU644O+4vcpd4=", + "status": "active" + }, + { + "algorithm": "ed25519", + "key_id": "registry-peer-b-key-1", + "peer_id": "registry-peer-b", + "public_key_base64": "TtMvY7818O7vyyXyii4fvchzrig1ZxsMlGD18S5FVqg=", + "status": "revoked" + }, + { + "algorithm": "ed25519", + "key_id": "registry-peer-b-key-2", + "peer_id": "registry-peer-b", + "public_key_base64": "iEuIV/TqoWE8YVBNs01L6vNGUXoOMd483dTZtCAdnQs=", + "status": "active" + } + ], + "registry_id": "example-org-checkpoints", + "schema_version": 1 +} diff --git a/benchmarks/templates/agent_cli_manifest.example.json b/benchmarks/templates/agent_cli_manifest.example.json new file mode 100644 index 0000000..7098d15 --- /dev/null +++ b/benchmarks/templates/agent_cli_manifest.example.json @@ -0,0 +1,21 @@ +{ + "arms": [ + "codex_cli", + "claude_code", + "morphic_control" + ], + "benchmark_id": "agent-cli-local-rehearsal", + "repetitions": 1, + "schema_version": 1, + "task": { + "checks": [ + "local_process" + ], + "goal": "Exercise the recorded benchmark pipeline without external agents.", + "handoff_assertions": [ + "receipt_contract" + ], + "id": "local-rehearsal", + "workspace_revision": "REPLACE_WITH_GIT_COMMIT" + } +} diff --git a/benchmarks/templates/agent_cli_recorder.example.json b/benchmarks/templates/agent_cli_recorder.example.json new file mode 100644 index 0000000..e2559ea --- /dev/null +++ b/benchmarks/templates/agent_cli_recorder.example.json @@ -0,0 +1,44 @@ +{ + "arm_commands": { + "claude_code": [ + "python3", + "-c", + "import json; print(json.dumps({'type':'system','subtype':'init','model':'local-rehearsal'})); print(json.dumps({'type':'result','subtype':'success','is_error':False,'result':'ok','usage':{'input_tokens':0,'output_tokens':0},'total_cost_usd':0.0}))" + ], + "codex_cli": [ + "python3", + "-c", + "import json; print(json.dumps({'type':'turn.completed','usage':{'input_tokens':0,'output_tokens':0}}))" + ], + "morphic_control": [ + "python3", + "-c", + "import json; print(json.dumps({'type':'morphic_benchmark_receipt','success':True,'model':'morphic-control[local-rehearsal]','usage':{'input_tokens':0,'output_tokens':0},'cost_usd':0.0}))" + ] + }, + "benchmark_id": "agent-cli-local-rehearsal", + "check_commands": { + "local_process": [ + "python3", + "-c", + "raise SystemExit(0)" + ] + }, + "estimated_cost_usd_per_trial": { + "claude_code": 0.0, + "codex_cli": 0.0, + "morphic_control": 0.0 + }, + "handoff_commands": { + "receipt_contract": [ + "python3", + "-c", + "raise SystemExit(0)" + ] + }, + "model_hints": { + "codex_cli": "local-rehearsal" + }, + "schema_version": 1, + "timeout_seconds": 10.0 +} diff --git a/benchmarks/templates/agent_cli_review_policy.example.json b/benchmarks/templates/agent_cli_review_policy.example.json new file mode 100644 index 0000000..d36ce83 --- /dev/null +++ b/benchmarks/templates/agent_cli_review_policy.example.json @@ -0,0 +1,10 @@ +{ + "benchmark_id": "agent-cli-local-rehearsal", + "minimum_distinct_reviewers": 2, + "operator_id": "REPLACE_WITH_OPERATOR_ID", + "reviewer_ids": [ + "REPLACE_WITH_REVIEWER_1_ID", + "REPLACE_WITH_REVIEWER_2_ID" + ], + "schema_version": 1 +} diff --git a/benchmarks/templates/agent_cli_reviewer_authority.example.json b/benchmarks/templates/agent_cli_reviewer_authority.example.json new file mode 100644 index 0000000..8e9cb48 --- /dev/null +++ b/benchmarks/templates/agent_cli_reviewer_authority.example.json @@ -0,0 +1,6 @@ +{ + "algorithm": "ed25519", + "authority_id": "REPLACE_WITH_ORG_BENCHMARK_CA", + "public_key_base64": "7UkoxijRwsbq6QM4kFmVYSlZJzpcY/k2NsFGFKyHN9E=", + "schema_version": 1 +} diff --git a/benchmarks/templates/agent_cli_reviewer_trust.example.json b/benchmarks/templates/agent_cli_reviewer_trust.example.json new file mode 100644 index 0000000..eef3d5e --- /dev/null +++ b/benchmarks/templates/agent_cli_reviewer_trust.example.json @@ -0,0 +1,21 @@ +{ + "benchmark_id": "agent-cli-local-rehearsal", + "keys": [ + { + "algorithm": "ed25519", + "key_id": "REPLACE_WITH_REVIEWER_1_KEY_ID", + "public_key_base64": "iojj3XQJ8ZX9UtstPLpdcspnCb8dlBIb83SIAbQPb1w=", + "reviewer_id": "REPLACE_WITH_REVIEWER_1_ID", + "status": "active" + }, + { + "algorithm": "ed25519", + "key_id": "REPLACE_WITH_REVIEWER_2_KEY_ID", + "public_key_base64": "gTl3Dqh9F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5Q=", + "reviewer_id": "REPLACE_WITH_REVIEWER_2_ID", + "status": "active" + } + ], + "review_policy_sha256": "b2b73f5017d25f3dab6b180cf9c8c2d248f9d341a0fde28c7ba392e103df2b09", + "schema_version": 1 +} diff --git a/benchmarks/templates/agent_cli_runtime_versions.example.json b/benchmarks/templates/agent_cli_runtime_versions.example.json new file mode 100644 index 0000000..9c3a076 --- /dev/null +++ b/benchmarks/templates/agent_cli_runtime_versions.example.json @@ -0,0 +1,18 @@ +{ + "benchmark_id": "agent-cli-local-rehearsal", + "runtimes": { + "claude_code": { + "executable": "python3", + "version": "REPLACE_WITH_PYTHON_VERSION" + }, + "codex_cli": { + "executable": "python3", + "version": "REPLACE_WITH_PYTHON_VERSION" + }, + "morphic_control": { + "executable": "python3", + "version": "REPLACE_WITH_PYTHON_VERSION" + } + }, + "schema_version": 1 +} diff --git a/benchmarks/templates/agent_cli_witness_trust.example.json b/benchmarks/templates/agent_cli_witness_trust.example.json new file mode 100644 index 0000000..4ac5458 --- /dev/null +++ b/benchmarks/templates/agent_cli_witness_trust.example.json @@ -0,0 +1,28 @@ +{ + "keys": [ + { + "algorithm": "ed25519", + "key_id": "witness-1-key-1", + "public_key_base64": "QwRr/kCSs+lJlOraFdzCDYqqB7ZY/TlU644O+4vcpd4=", + "status": "active", + "witness_id": "witness-1" + }, + { + "algorithm": "ed25519", + "key_id": "witness-2-key-1", + "public_key_base64": "TtMvY7818O7vyyXyii4fvchzrig1ZxsMlGD18S5FVqg=", + "status": "active", + "witness_id": "witness-2" + }, + { + "algorithm": "ed25519", + "key_id": "witness-3-key-1", + "public_key_base64": "iEuIV/TqoWE8YVBNs01L6vNGUXoOMd483dTZtCAdnQs=", + "status": "active", + "witness_id": "witness-3" + } + ], + "log_id": "example-org-agent-cli", + "minimum_distinct_witnesses": 2, + "schema_version": 1 +} diff --git a/docs/AGENT_CLI.md b/docs/AGENT_CLI.md index b87f82c..fc5a25c 100644 --- a/docs/AGENT_CLI.md +++ b/docs/AGENT_CLI.md @@ -228,6 +228,983 @@ class CodexCLIDriver: --- +## Same-task comparative evidence + +Morphicの優位性は主観的な総合点ではなく、同一課題・同一workspace revision・ +同一verification checks・同一反復数で比較する。記録済み試行は次で評価する。 + +```bash +morphic benchmark agent-cli \ + --manifest benchmark-manifest.json \ + --results benchmark-results.json + +# CI artifact向けのstable JSON +morphic benchmark agent-cli \ + --manifest benchmark-manifest.json \ + --results benchmark-results.json \ + --json +``` + +manifestは3 arms (`codex_cli`, `claude_code`, `morphic_control`)を必須とし、taskに +`id`, `goal`, `workspace_revision`, `checks`, `handoff_assertions`を宣言する。 +resultsは各arm × trialをexactly once記録し、completion、accepted patch、通過した +checks/handoff assertions、elapsed time、cost、human interventions、recoveryを持つ。 + +評価器は以下をarm別に算出し、metricごとのleaderを示す。 + +- completion rate / accepted patch rate +- verification rate（宣言済みchecksから算出） +- median elapsed seconds / mean cost / mean human interventions +- recovery rate +- context-handoff fidelity（宣言済みhandoff assertionsから算出） + +恣意的なweightを避けるためcomposite scoreは作らない。出力にはtimestampを含めず、 +JSON keyをsortして同じ入力からbyte-stableなartifactを作る。このコマンドはファイルを +読むだけでnative engineやpaid APIを起動しない。将来のlive recorderはcost capを伴う +別のexplicit opt-inとして追加する。 + +### Isolated trial recorder + +Phase 41のrecorderはdefaultでread-only planだけを返す。manifestとrecorder configを +検証し、trial count、command fingerprints、設定上の最大費用見積りを出すが、agent、 +Git worktree、verification commandは起動しない。 + +```bash +# Read-only plan. No worktree or agent process is created. +morphic benchmark agent-cli-record \ + --manifest benchmark-manifest.json \ + --config recorder-config.json \ + --json + +# Explicit live execution. worktree root must be outside the source repository. +morphic benchmark agent-cli-record \ + --manifest benchmark-manifest.json \ + --config recorder-config.json \ + --source-root . \ + --worktree-root ../morphic-benchmark-worktrees \ + --evidence benchmark-evidence.json \ + --execute \ + --acknowledge-paid \ + --cost-cap-usd 3.00 \ + --json +``` + +recorder configは3 armの`arm_commands`と`estimated_cost_usd_per_trial`、manifestと +exact matchする`check_commands` / `handoff_commands`、1 commandあたりの +`timeout_seconds`を持つ。argvは配列で指定し、`{goal}`, `{workspace}`, `{arm}`, +`{trial}`を使用できる。shell展開は行わない。 + +実行には`--execute`、`--acknowledge-paid`、全trialの最大見積りを覆う +`--cost-cap-usd`がすべて必要。各arm/trialはpinned revisionの別detached worktreeで +実行され、正常・失敗・例外を問わずcleanupする。evidenceはargv/stdout/stderrの +SHA-256、byte数、exit code、timeout、elapsed time、check/assertion結果だけを保存し、 +raw prompt/outputは保存しない。既存evidenceは上書きしない。 + +このcost capは設定見積りに対する事前authorization gateであり、provider請求額を +process内でhard-stopするものではない。実費とaccepted-patch判定は +`pending_adjudication`として残す。provider receipt parserとreview adjudicatorが +揃うまでは、recorder evidenceをPhase 40の最終comparison resultへ自動変換しない。 + +### Receipt normalization and adjudication + +Phase 42ではrecorderがagent stdoutを保持している瞬間だけreceiptを解析し、raw出力を +捨てる前に以下のnormalized fieldsへ変換する。 + +- provider / success / model +- non-negative token usage +- cost USD / cost source +- parse error count + +CodexはJSONL usageと`model_hints.codex_cli`から既存のcost calculatorで再計算し、 +receipt costが一致しない場合は拒否する。Claudeはstream-json resultの +`total_cost_usd`をprovider-reported valueとして保持する。Morphic-controlled commandは +最後に次のcanonical envelopeを出力する必要がある。 + +```json +{ + "type": "morphic_benchmark_receipt", + "success": true, + "model": "o4-mini", + "usage": {"input_tokens": 120, "output_tokens": 30}, + "cost_usd": 0.02 +} +``` + +全trialでreceiptが得られた場合だけevidenceの`cost_collection`が +`normalized_receipts`になる。欠損を0ドルとして補完しない。 + +independent review bundleは各arm/trialについてagent argv fingerprint、accepted patch、 +human interventions、recovery、reviewer id、review artifact SHA-256を記録する。次のoffline +commandがmachine evidenceとreview bundleを結合する。 + +```bash +morphic benchmark agent-cli-finalize \ + --manifest benchmark-manifest.json \ + --evidence benchmark-evidence.json \ + --reviews benchmark-reviews.json \ + --output benchmark-results.json \ + --json + +morphic benchmark agent-cli \ + --manifest benchmark-manifest.json \ + --results benchmark-results.json \ + --json +``` + +finalizerは完全なarm/trial matrix、task/revision identity、provider、argv fingerprint、 +check/handoff evidence、receipt parse status、review consistency、actual-cost totalを検証する。 +失敗trialをacceptedにするreview、authorized cap超過、既存outputへの上書きは拒否する。 +この処理はagentやpaid APIを起動せず、同じ入力からtimestamp-free/sorted-key resultを作る。 + +### First-party Morphic receipt and zero-cost rehearsal + +Morphic-controlled armはwrapperなしでcanonical receiptを出力できる。通常のone-shot出力を +維持し、最後のstdout lineだけをreceiptにする。 + +```bash +morphic code \ + --benchmark-receipt \ + --workspace . \ + "Implement the benchmark task" +``` + +`--benchmark-receipt`はcouncil turnのcostを合算し、normalized completion eventに含まれる +non-negative usage counterだけを集約する。model fieldは +`morphic-control[]`となる。実行失敗またはCtrl-Cでは、未確定費用を +0ドルと偽らずreceiptを出さないため、recorder/finalizerは欠損としてfail closedする。 +通常のflagなし出力は変えない。 + +Phase 43のlocal rehearsalは外部agentやAPIを起動せず、Phase 41-42の全経路を検査する。 + +```bash +morphic benchmark agent-cli-rehearse \ + --source-root . \ + --revision HEAD \ + --output-dir ../agent-cli-rehearsal +``` + +このコマンドが使うarm commandは内部生成された`python -c` fixtureだけで、利用者が +Codex/Claude commandへ差し替えるoptionはない。configured estimateとnormalized actual +costはともに0ドル。pinned detached worktreeで3 armを通し、次のbundleを新規directoryへ +exclusiveに発行する。 + +- `manifest.json` +- `recorder-config.json` +- `evidence.json` +- `reviews.json` +- `results.json` + +rehearsal reviewは`accepted_patch=false`に固定される。これはreceipt/parser/join/isolationの +動作確認であり、agent品質比較ではない。既存output directoryは上書きしない。read-only +planの編集開始点として`benchmarks/templates/agent_cli_manifest.example.json`と +`agent_cli_recorder.example.json`も同梱する。実キャンペーンは従来どおり別の +`agent-cli-record --execute --acknowledge-paid --cost-cap-usd ...`による明示承認が必要。 + +### Campaign preflight and bound reviews + +Phase 44では実キャンペーン前にimmutable revision、runtime declarations、全commandを +1つのnon-authorizing artifactへ固定する。runtime versionは利用者が収集してJSONへ記入し、 +Morphic自身は`--version`を含むagent commandを実行しない。 + +```bash +morphic benchmark agent-cli-preflight \ + --manifest benchmark-manifest.json \ + --config recorder-config.json \ + --runtime-versions runtime-versions.json \ + --source-root . \ + --output benchmark-preflight.json \ + --json +``` + +manifestの`workspace_revision`はsymbolic `HEAD`ではなく、Gitで解決できるfull lowercase +40-character commit hashでなければならない。runtime declarationは3 armをexactly once持ち、 +各`executable`がrecorder configのarm command先頭と一致する必要がある。version stringは +whitespace-normalize後にSHA-256化される。arm/check/handoff commandもPhase 41と同じ方法で +fingerprintされる。raw goalを公開せずにgoal/timeout/model hintを含む完全な契約変更を +検出するため、manifest全体とconfig全体のcanonical SHA-256も保持する。artifact自身の +SHA-256と`execution_authorized=false`を含み、preflight成功だけでagent実行は許可されない。 + +recording後はindependent reviewer用の未記入templateを生成する。 + +```bash +morphic benchmark agent-cli-review-template \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --output benchmark-reviews.json \ + --json +``` + +templateは全arm/trialを持ち、human judgment fieldsはすべて`null`、 +`review_completed=false`である。reviewerはaccepted patch、interventions、recovery、reviewer id、 +review artifact SHA-256を埋め、最後に`review_completed=true`へ変更する。preflight/evidence +SHA-256とagent argv SHA-256は変更しない。 + +```bash +morphic benchmark agent-cli-finalize \ + --manifest benchmark-manifest.json \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --reviews benchmark-reviews.json \ + --output benchmark-results.json +``` + +bound reviewを`--preflight`なしでfinalizeすること、別evidenceへ流用すること、fingerprintを +変更することは拒否される。Phase 42形式のbinding fieldを持たないlegacy reviewは後方互換で +利用できる。`benchmarks/templates/agent_cli_runtime_versions.example.json`を編集開始点として +同梱する。 + +### Reviewer separation and campaign status + +Phase 45ではoperatorとreviewerの構造的分離をpolicy declarationで固定する。 +`benchmarks/templates/agent_cli_review_policy.example.json`をコピーし、operator ID、許可する +reviewer IDs、必要なminimum distinct reviewer数を記入する。 + +```json +{ + "schema_version": 1, + "benchmark_id": "campaign-001", + "operator_id": "operator-1", + "reviewer_ids": ["reviewer-1", "reviewer-2"], + "minimum_distinct_reviewers": 2 +} +``` + +このpolicyをreview template生成とfinalizeの両方へ渡す。 + +```bash +morphic benchmark agent-cli-review-template \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --review-policy reviewer-policy.json \ + --output benchmark-reviews.json + +morphic benchmark agent-cli-finalize \ + --manifest benchmark-manifest.json \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --reviews benchmark-reviews.json \ + --review-policy reviewer-policy.json \ + --output benchmark-results.json +``` + +policyはreviewer IDsをsortしてcanonical SHA-256を作り、review artifactへbindする。 +operator自身のreview、allowlist外ID、minimum distinct reviewer未達、decision数より大きく +実現不能なminimumは拒否する。これは宣言されたIDの構造的分離であり、本人確認や署名を +意味しない。 + +campaignの現在位置はartifactを変更せず確認できる。 + +```bash +morphic benchmark agent-cli-status \ + --manifest benchmark-manifest.json \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --reviews benchmark-reviews.json \ + --review-policy reviewer-policy.json \ + --results benchmark-results.json \ + --json +``` + +status stageは`manifest_ready` → `preflight_ready` → `recorded` → `review_pending` → +`review_complete` → `finalized`。途中artifactの欠落、別manifest/evidenceの混入、estimate、 +policy、review、resultsの不一致はfail closedになる。このcommandはファイルを読むだけで、 +全stageにおいて`paid_execution_authorized=false`を返す。 + +### Signed reviewer attestations + +Phase 46では、policyのreviewer IDをEd25519公開鍵へ結び付ける。まず +`benchmarks/templates/agent_cli_reviewer_trust.example.json`をコピーし、example公開鍵を +各reviewerが管理する実鍵へ必ず置き換える。trust declarationはreview policy SHA-256、 +reviewer ID、key ID、公開鍵、`active` / `revoked` statusをcanonical SHA-256へ固定する。 +key rotation時は旧鍵を`revoked`で残し、新しいactive keyを追加する。 + +trust-bound review templateを生成し、reviewerがdecisionを完了した後、canonical signing +payloadを生成する。 + +```bash +morphic benchmark agent-cli-review-template \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --review-policy reviewer-policy.json \ + --reviewer-trust reviewer-trust.json \ + --output benchmark-reviews.json + +# reviewerがbenchmark-reviews.jsonを完成させ、review_completed=trueにした後 +morphic benchmark agent-cli-attestation-template \ + --reviews benchmark-reviews.json \ + --review-policy reviewer-policy.json \ + --reviewer-trust reviewer-trust.json \ + --output benchmark-attestation-template.json \ + --json +``` + +attestation templateはdistinct reviewerごとに1つのstatementと +`signing_payload_base64`を出す。statementはbenchmark/task/revision、preflight、evidence、 +review policy、reviewer trust、completed reviews全体、当該reviewerのdecision集合をbindする。 +Morphicは秘密鍵を読まず、reviewerはpayloadを自身のEd25519秘密鍵で外部署名し、署名とkey IDを +`ReviewAttestationBundle`へ格納する。 + +```bash +morphic benchmark agent-cli-finalize \ + --manifest benchmark-manifest.json \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --reviews benchmark-reviews.json \ + --review-policy reviewer-policy.json \ + --reviewer-trust reviewer-trust.json \ + --attestations benchmark-attestations.json \ + --output benchmark-results.json +``` + +trust-bound reviewはdistinct reviewer全員の署名が揃わない限りfinalizeできない。unknown key、 +revoked key、invalid signature、欠落reviewer、別review/evidence/policy/trustからの混入は拒否する。 +statusには`review_attestation_pending`が加わり、検証後だけ`review_complete`へ進む。unsigned legacy +campaignは従来の6段階とfinalize behaviorを維持する。署名は登録済み秘密鍵の保有を証明するが、 +trust declarationのkey enrollment自体は実在人物の本人確認ではない。組織CA、OIDC/Sigstore、 +または外部key directoryとの結合は次段階である。 + +### Organization-authority anchored campaigns + +Phase 47では、offline Ed25519 organization authorityをout-of-band trust anchorとして追加する。 +`agent_cli_reviewer_authority.example.json`と`agent_cli_anchored_reviewer_trust.example.json`を +開始点にできるが、同梱example公開鍵は実運用で必ず組織管理鍵へ置き換える。authority artifactは +authority ID、algorithm、public key、public-key SHA-256、self fingerprintを固定する。秘密鍵は +Morphicへ渡さない。 + +anchored trustを作成後、全reviewer keyについてCA署名payloadを生成する。 + +```bash +morphic benchmark agent-cli-reviewer-enrollment-template \ + --review-policy reviewer-policy.json \ + --reviewer-trust anchored-reviewer-trust.json \ + --reviewer-authority reviewer-authority.json \ + --output reviewer-enrollment-template.json \ + --json +``` + +organization authorityは各`signing_payload_base64`を外部署名し、statementとsignatureを +`ReviewerEnrollmentBundle`へ格納する。statementはauthority、benchmark、review policy、 +exact reviewer trust、reviewer/key ID、reviewer public-key fingerprintをbindする。trust内の +active/revokedを含む全鍵がexactly once CA enrollmentされなければ検証は失敗する。 + +authority-bound finalizeは通常のreview attestationsに加えてauthorityとenrollmentsを要求する。 + +```bash +morphic benchmark agent-cli-finalize \ + --manifest benchmark-manifest.json \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --reviews benchmark-reviews.json \ + --review-policy reviewer-policy.json \ + --reviewer-trust anchored-reviewer-trust.json \ + --reviewer-authority reviewer-authority.json \ + --reviewer-enrollments reviewer-enrollments.json \ + --attestations benchmark-attestations.json \ + --output benchmark-results.json +``` + +results生成後、全artifactを1つのauthority signing payloadへ固定する。 + +```bash +morphic benchmark agent-cli-campaign-envelope-template \ + --manifest benchmark-manifest.json \ + --preflight benchmark-preflight.json \ + --evidence benchmark-evidence.json \ + --reviews benchmark-reviews.json \ + --review-policy reviewer-policy.json \ + --reviewer-trust anchored-reviewer-trust.json \ + --reviewer-authority reviewer-authority.json \ + --reviewer-enrollments reviewer-enrollments.json \ + --attestations benchmark-attestations.json \ + --results benchmark-results.json \ + --output campaign-envelope-template.json +``` + +envelopeはmanifest、preflight、evidence、reviews、policy、trust、enrollments、attestations、 +resultsのSHA-256とidentityをbindし、`paid_execution_authorized=false`を固定する。authorityが +payloadを外部署名した`SignedCampaignEnvelope`をstatusへ渡した場合だけauthority-bound campaignは +`finalized`になる。署名前は`campaign_envelope_pending`、CA enrollment不足時は +`reviewer_enrollment_pending`を返す。unanchored Phase 46とunsigned legacy campaignは従来どおり。 + +このoffline CA経路はoperatorだけが作ったreviewer鍵を排除できる。ただしauthority root公開鍵の +安全なout-of-band配布、certificate expiry、root revocation/rotation、transparency logは別契約であり、 +現在のartifactだけでは保証しない。 + +### Authority-root continuity and campaign transparency + +Phase 48ではrootをversioned ledgerとして扱う。generation 1はout-of-band genesisで、generation 2以降は +直前rootが`generation`、predecessor SHA-256、successor SHA-256を外部署名する。Morphicは秘密鍵を読まず、 +rotation signing requestを生成する。 + +```bash +morphic benchmark agent-cli-authority-rotation-template \ + --generation 2 \ + --predecessor predecessor-authority.json \ + --successor successor-authority.json \ + --output rotation-request.json +``` + +署名済みrotation certificateをgeneration列へ格納後、active rootが署名するledger payloadを作る。 +`--generations`は`generations`配列とoptional `revoked_authority_sha256`配列を持つJSON objectである。 + +```bash +morphic benchmark agent-cli-authority-root-ledger-template \ + --generations authority-generations.json \ + --output authority-root-ledger-request.json +``` + +ledgerはcontiguous generation、root非再利用、各predecessor署名、既知rootだけのrevocation、非revoked +active root、active-root ledger署名を検証する。ledger-bound reviewer trustは +`reviewer_authority_sha256`にactive root、`authority_root_ledger_sha256`にexact ledgerを指定する。 +finalizeとcampaign envelope templateには`--authority-root-ledger`を渡す。fieldを持たないPhase 47 artifactは +fingerprintと署名bytesを変更せず従来経路を維持する。 + +透明性ログはcomplete entry arrayからoffline生成できる。 + +```bash +morphic benchmark agent-cli-transparency-log \ + --log-id example-org-agent-cli \ + --entries transparency-entries.json \ + --output transparency-log.json + +morphic benchmark agent-cli-transparency-tree-head-template \ + --log transparency-log.json \ + --authority-root-ledger signed-authority-root-ledger.json \ + --output tree-head-request.json + +morphic benchmark agent-cli-transparency-proof \ + --log transparency-log.json \ + --tree-head signed-tree-head.json \ + --leaf-index 2 \ + --output campaign-envelope-proof.json +``` + +Merkle treeはRFC 6962と同じdomain separation、すなわちleafを +`SHA256(0x00 || canonical_entry)`、nodeを`SHA256(0x01 || left || right)`で構築する。tree headは +active rootが外部署名する。ledger-bound campaignはexact campaign envelope SHA-256のinclusion proofを +`agent-cli-status --transparency-proof ...`で検証するまで`transparency_pending`であり、検証後だけ +`finalized`になる。旧/new complete logのappend-only検証は旧entriesが新logのexact prefixであることを +要求する。 + +この契約は初回genesis鍵の安全な配布やcompromise後のout-of-band trust resetを代替しない。 +certificate expiry、OIDC/Sigstore identityもまだ実装しない。どのCLIもpaid executionを許可しない。 + +### Compact consistency and witness checkpoints + +Phase 49では、complete旧logを配布せずにappend-only growthを検証できる。current complete log、旧/newの +active-root-signed tree head、root ledgerからRFC 6962 minimal consistency proofを作る。 + +```bash +morphic benchmark agent-cli-transparency-consistency-proof \ + --current-log transparency-log-current.json \ + --previous-tree-head signed-tree-head-previous.json \ + --current-tree-head signed-tree-head-current.json \ + --authority-root-ledger signed-authority-root-ledger.json \ + --output consistency-proof.json +``` + +proofは`SUBPROOF` recursionが返す最大`ceil(log2(n)) + 1`個のSHA-256 nodeだけを保持する。検証側は +complete logなしで旧rootとnew rootを同時に再構成し、log ID、tree size、root ledger、tree-head署名、 +proof fingerprint、audit path、両rootのどれかが一致しなければ拒否する。 + +単一log authorityのequivocationへ追加の監視境界を置く場合は、exampleを組織ごとのwitness公開鍵へ +置き換えてtrustを正規化する。 + +```bash +morphic benchmark agent-cli-witness-trust \ + --declaration benchmarks/templates/agent_cli_witness_trust.example.json \ + --output witness-trust.json + +morphic benchmark agent-cli-witness-checkpoint-template \ + --witness-trust witness-trust.json \ + --consistency-proof consistency-proof.json \ + --authority-root-ledger signed-authority-root-ledger.json \ + --output witness-checkpoint-template.json +``` + +witness trustはwitness/key ID、Ed25519 public key、active/revoked status、minimum distinct countを +self-fingerprintする。minimumはactive witness総数のstrict majorityでなければならず、任意の2 accepted +quorumが少なくとも1 witnessで交差する。各witnessは同じcheckpoint statementを外部署名する。statementは +old/new sizeとroot、両tree-head fingerprint、root ledger、consistency proof、witness trustをbindする。 +Morphicは秘密鍵を読まない。 + +opt-in witness pathでは`agent-cli-status`へ次を追加する。 + +```bash + --transparency-consistency-proof consistency-proof.json \ + --transparency-witness-trust witness-trust-declaration.json \ + --witness-checkpoint signed-witness-checkpoint.json +``` + +inclusion proofだけが揃った状態は`witness_pending`となり、compact consistencyとstrict-majority witness +signaturesを検証後だけ`finalized`になる。witness inputを指定しないPhase 48 campaignは従来どおり +inclusion verificationでfinalizeできる。同じlog IDとtree sizeに異なるwitnessed rootが存在した場合は +split viewとして拒否する。 + +### Durable checkpoint registry and authenticated peer exchange + +Phase 50では、検証済みwitness checkpointをlocal append-only registryへ保存する。recordはregistry ID、 +sequence、previous record SHA-256、root ledger、witness trust、consistency proof、checkpointをbindして +self-fingerprintする。storeはappend前とstatus replay時に全recordを再検証する。 + +```bash +morphic benchmark agent-cli-checkpoint-registry-store \ + --registry checkpoint-registry.jsonl \ + --registry-id example-org-checkpoints \ + --consistency-proof consistency-proof.json \ + --witness-checkpoint signed-witness-checkpoint.json \ + --witness-trust witness-trust.json \ + --authority-root-ledger signed-authority-root-ledger.json + +morphic benchmark agent-cli-checkpoint-registry-status \ + --registry checkpoint-registry.jsonl \ + --registry-id example-org-checkpoints \ + --witness-trust witness-trust.json \ + --authority-root-ledger signed-authority-root-ledger.json \ + --json +``` + +appendはprocess間file lock、`O_APPEND`、`fsync`、mode 0600を使用する。同じproof/checkpointのretryは +同じrecordを返す。sequence gap、previous hash改ざん、record改ざん、truncated tail、現在headへ接続しない +stale proof、同じsizeの異なるwitnessed rootは書き込み前または次回replayで拒否する。statusはregistryが +存在しない場合もfileを作らない。 + +peer公開鍵はexampleをdeployment固有の鍵へ置き換え、active rotation keyをpeerごとに最低1つ残す。 + +```bash +morphic benchmark agent-cli-checkpoint-peer-trust \ + --declaration benchmarks/templates/agent_cli_checkpoint_peer_trust.example.json \ + --output checkpoint-peer-trust.json + +morphic benchmark agent-cli-checkpoint-registry-export-template \ + --registry checkpoint-registry.jsonl \ + --registry-id example-org-checkpoints \ + --witness-trust witness-trust.json \ + --authority-root-ledger signed-authority-root-ledger.json \ + --peer-trust checkpoint-peer-trust.json \ + --source-peer-id registry-peer-a \ + --output checkpoint-exchange-request.json +``` + +exportはlatest recordをdefaultとし、`--sequence`で過去recordを選べる。requestのcanonical signing bytesを +eligible Ed25519 keyで外部署名し、exact recordと署名を`SignedCheckpointExchangePacket`へ格納する。 +受信側はsource peer/keyのactive trust、packet/record fingerprint、署名、registry IDを検証してから、local +headに対する同じlocked append経路でimportする。 + +```bash +morphic benchmark agent-cli-checkpoint-registry-import \ + --registry peer-checkpoint-registry.jsonl \ + --registry-id example-org-checkpoints \ + --packet signed-checkpoint-packet.json \ + --peer-trust checkpoint-peer-trust.json \ + --witness-trust witness-trust.json \ + --authority-root-ledger signed-authority-root-ledger.json +``` + +これはtransport-neutralなlocal persistence/exchange contractであり、online listener、peer discovery、 +real-world peer identity、global consensusは提供しない。1 packetは1 exact recordを運ぶため、遅れたpeerの +multi-record catch-upは現時点ではsequence順にpacketを交換する。private keyは全CLIで読み込まない。 + +### Authenticated range sync and durable peer cursors + +Phase 51では、最大1000 recordまでのbounded contiguous rangeを1つのpeer署名でcatch-upできる。 +`--max-records`のdefaultは100、`--start-sequence`のdefaultは0である。 + +```bash +morphic benchmark agent-cli-checkpoint-range-export-template \ + --registry checkpoint-registry.jsonl \ + --registry-id example-org-checkpoints \ + --witness-trust witness-trust.json \ + --authority-root-ledger signed-authority-root-ledger.json \ + --peer-trust checkpoint-peer-trust.json \ + --source-peer-id registry-peer-a \ + --start-sequence 0 \ + --max-records 100 \ + --output checkpoint-range-request.json +``` + +range statementはbase previous-record hash、first/last sequenceとrecord hash、全record fingerprint列、 +registry ID、peer trustをbindする。eligible peer keyで外部署名してrecordsとともに +`SignedCheckpointRangeBundle`へ格納する。受信側は次のcommandでimportする。 + +```bash +morphic benchmark agent-cli-checkpoint-range-import \ + --registry peer-checkpoint-registry.jsonl \ + --registry-id example-org-checkpoints \ + --range-bundle signed-checkpoint-range.json \ + --peer-trust checkpoint-peer-trust.json \ + --witness-trust witness-trust.json \ + --authority-root-ledger signed-authority-root-ledger.json +``` + +importはpeer signatureとrange内部chain、全recordのauthority/witness/Merkle bindingを先に検証する。 +exclusive lock取得後に既存sequenceとのoverlapをexact record単位で比較し、missing contiguous suffixだけを +1 batchでappend + `fsync`する。gap、conflicting overlap、stale/forked proofは書込み前に拒否する。 +process中のwrite failureはoriginal file sizeへtruncateする。process crash/power lossによるpartial tailは +次回replayでfail closedになるが、filesystem-level transactionとは表現しない。 + +rangeを現在headまで適用した受信peerはack signing requestを生成する。 + +```bash +morphic benchmark agent-cli-checkpoint-acknowledgement-template \ + --registry peer-checkpoint-registry.jsonl \ + --registry-id example-org-checkpoints \ + --range-bundle signed-checkpoint-range.json \ + --peer-trust checkpoint-peer-trust.json \ + --witness-trust witness-trust.json \ + --authority-root-ledger signed-authority-root-ledger.json \ + --acknowledging-peer-id registry-peer-b \ + --output checkpoint-acknowledgement-request.json +``` + +ackはsource/receiver peer、exact range bundle、applied record sequence/hash、tree size/rootをbindする。 +receiverが外部署名した`SignedCheckpointAcknowledgement`をsourceへ返し、sourceはcursor ledgerへ保存する。 + +```bash +morphic benchmark agent-cli-checkpoint-cursor-store \ + --cursor-ledger checkpoint-peer-cursors.jsonl \ + --registry-id example-org-checkpoints \ + --acknowledgement signed-checkpoint-acknowledgement.json \ + --peer-trust checkpoint-peer-trust.json + +morphic benchmark agent-cli-checkpoint-cursor-status \ + --cursor-ledger checkpoint-peer-cursors.jsonl \ + --registry-id example-org-checkpoints \ + --peer-trust checkpoint-peer-trust.json \ + --json +``` + +cursor ledgerもmode 0600、file lock、hash chain、`O_APPEND`、`fsync`を使用する。source/receiver pairごとに +acknowledged sequenceは単調増加し、exact retryだけ冪等化する。regression、同一sequenceの別record、 +signature/fingerprint/hash-chain tamperingは拒否する。 + +現時点ではartifactを運ぶnetwork listener、peer discovery、global consensusを提供しない。またackは +exact peer-trust fingerprintへbindされるため、trust更新後にhistorical cursorをreplayするには旧trust +artifactが必要である。次のtrust sliceではpeer-trust generation ledgerとsigned rollover continuityを追加する。 + +### Peer-trust generation and rollover continuity + +Phase 52では、旧trust artifactをoperatorが手動選択する代わりに、out-of-band genesisから署名で連結した +peer-trust generation ledgerを使用できる。successor trustを用意し、直前trustのactive peer向けsigning +requestを生成する。 + +```bash +morphic benchmark agent-cli-checkpoint-peer-trust-rotation-template \ + --predecessor-peer-trust checkpoint-peer-trust-v1.json \ + --successor-peer-trust checkpoint-peer-trust-v2.json \ + --generation 2 \ + --output checkpoint-peer-trust-rotation-request.json +``` + +required quorumはpredecessorのdistinct active peer数から`floor(n / 2) + 1`として自動計算する。 +各peerは同一statementをeligible active keyで外部署名する。statementはregistry、generation、 +predecessor/successor trust fingerprint、required quorumをbindする。署名を +`CheckpointPeerTrustRotationCertificate`へ格納後、generation列からledgerを構築する。 + +```bash +morphic benchmark agent-cli-checkpoint-peer-trust-ledger \ + --generations checkpoint-peer-trust-generations.json \ + --output checkpoint-peer-trust-ledger.json +``` + +`--generations`は`generations`配列を持つJSON objectである。generation 1はrotationなしのgenesis、 +generation 2以降はtrustと直前世代が承認したrotation certificateを持つ。検証はcontiguous order、 +registry不変、trust非再利用、exact predecessor/successor、strict-majority distinct peers、active predecessor +key、全Ed25519署名、certificate/ledger fingerprintを毎回確認する。 + +cursor ledgerは単一snapshotの代わりにgeneration ledgerを使用できる。 + +```bash +morphic benchmark agent-cli-checkpoint-cursor-status \ + --cursor-ledger checkpoint-peer-cursors.jsonl \ + --registry-id example-org-checkpoints \ + --peer-trust-ledger checkpoint-peer-trust-ledger.json \ + --json +``` + +`cursor-store`も同じ`--peer-trust-ledger`を受け付ける。`--peer-trust`と`--peer-trust-ledger`はexactly oneを +指定する。ackがbindするtrust fingerprintをledgerから解決するため、rotation前後のackを同一cursor chainで +検証できる。revoked keyはsuccessor generationの新artifact署名には使えないが、predecessor generation時点で +正当に署名されたhistorical ackはその世代のtrustで検証される。 + +genesis trustの初回配布、最新ledger fingerprintのout-of-band pinning、古いがvalidなledgerへのrollback検出、 +real-world peer identity、network transportは別境界である。次はexplicit opt-in loopback transportでprotocol +version、challenge nonce、replay防止、size/timeout制限を固定してからremote/TLSへ広げる。 + +### Authenticated loopback checkpoint gossip + +Phase 53では、事前署名済みcheckpoint rangeとsigned acknowledgementを明示的opt-inのloopback transportで +交換できる。server用JSONは`bundles`配列に`SignedCheckpointRangeBundle`を格納する。各bundleは起動前に +peer trustで検証されるため、listenerがprivate keyを読むことはない。 + +```bash +morphic benchmark agent-cli-checkpoint-gossip-serve \ + --descriptor .morphic/gossip/peer-1.json \ + --range-bundles signed-checkpoint-ranges.json \ + --cursor-ledger checkpoint-peer-cursors.jsonl \ + --registry-id example-org-checkpoints \ + --source-peer-id peer-1 \ + --peer-trust-ledger checkpoint-peer-trust-ledger.json \ + --max-requests 64 \ + --lifetime-seconds 300 +``` + +listenerは`127.0.0.1`のrandom portだけへbindする。descriptor parentは0700、descriptorは0600で、protocol +version、registry、source peer、random instance id、32-byte bearer tokenを保持する。既存descriptorは上書き +しない。各接続はone-use 32-byte client nonceとserver nonceを交換し、protocol、instance、registry、source +peer、operation、payloadをHMAC-SHA256へbindする。challenge responseと最終responseも同じtokenで認証する。 + +clientはstatusで配布可能なexact rangeを確認し、開始sequenceが一致する事前署名済みbundleを取得する。 + +```bash +morphic benchmark agent-cli-checkpoint-gossip-status \ + --descriptor .morphic/gossip/peer-1.json \ + --json + +morphic benchmark agent-cli-checkpoint-gossip-fetch \ + --descriptor .morphic/gossip/peer-1.json \ + --start-sequence 0 \ + --max-records 100 \ + --peer-trust-ledger checkpoint-peer-trust-ledger.json \ + --output received-range.json \ + --json +``` + +fetch clientはtransport HMACだけを信用せず、受信bundleのpeer Ed25519署名とexact trust fingerprintを再検証 +する。その後、既存`agent-cli-checkpoint-range-import`へ渡すことでauthority root、witness quorum、Merkle +consistency、registry overlap/hash chainまで検証してatomic importする。fetchだけではregistryへ書き込まない。 + +receiverが既存acknowledgement-templateを外部署名した後、source peerへ返せる。 + +```bash +morphic benchmark agent-cli-checkpoint-gossip-ack \ + --descriptor .morphic/gossip/peer-1.json \ + --acknowledgement signed-acknowledgement.json \ + --json +``` + +serverはackのpeer署名、registry/source binding、trust generationを検証し、既存mode 0600・locked・ +hash-chained cursor storeのmonotonicity規則を通した場合だけ保存する。同一ack retryは冪等で、不正署名や +regression/conflictは書き込み前に拒否する。 + +protocol limitは64 KiB request、2 MiB response、8 concurrent clients、最大1,024 retained noncesと +authenticated requests、各read/dispatch 2秒である。`--max-requests`到達、`--lifetime-seconds`、cancel、 +context終了時はlistenerを閉じ、active writer/taskを終了し、instance/tokenが一致するowned descriptorだけを +削除する。 + +descriptor tokenは同一host上の短命transportだけを認証する。remote bind、TLS/mTLS、peer discovery、 +automatic private-key signing、常駐daemon、複数rangeを自動追跡するpull loopは実装していない。次は +trust-ledger rollback pinningを伴うbounded/resumable catch-up loopとdurable audit stateを追加してから、 +remote mTLSへ進む。 + +### Durable bounded gossip catch-up loop + +Phase 54では、Phase 53のstatus/fetchをbounded loopとして実行し、verified local registryの次sequenceから +再開できる。peer-trust snapshotではなくgeneration ledgerを必須にし、最新確認済みgenerationをdurable +sync auditへpinする。 + +```bash +morphic benchmark agent-cli-checkpoint-gossip-sync \ + --descriptor .morphic/gossip/peer-1.json \ + --registry checkpoint-registry.jsonl \ + --sync-audit checkpoint-sync-peer-1.jsonl \ + --registry-id example-org-checkpoints \ + --source-peer-id peer-1 \ + --peer-trust-ledger checkpoint-peer-trust-ledger.json \ + --witness-trust witness-trust.json \ + --authority-root-ledger authority-root-ledger.json \ + --max-rounds 16 \ + --max-records 1000 \ + --max-attempts 3 \ + --json +``` + +loopはauthenticated statusのavailable rangeから、`first_sequence <= next local sequence <= last_sequence`を +満たす最も新しいstartを選ぶ。fetch後にstatusのbundle fingerprintと一致することを確認し、peer trust +generationからEd25519 keyを解決する。その後、既存import pathでauthority root、witness quorum、Merkle +consistency、overlap、hash chainを再検証し、missing suffixだけをatomic appendする。既存local prefixの再取得は +許容するが、exact overlapでなければ0 byte mutationで失敗する。 + +sync auditはmode 0600 JSONLで、1回のloop全体に`flock(LOCK_EX | LOCK_NB)`を保持する。同じauditに対する +多重loopは待たずに拒否する。各recordはsequence、previous hash、registry/source、verified local head、 +peer-trust generation/trust/ledger fingerprint、exact loop-policy fingerprintをbindし、次のeventだけを記録する。 + +- `imported`: range fingerprint、first/last sequence、新規import件数 +- `retry`: `status`または`fetch_range`とattempt番号のみ +- `recovered`: registry append後・audit append前に停止した状態の回復 +- `trust_advanced`: contiguous trust ledger extensionのpin前進 +- `stopped`: machine-readableな停止理由 + +停止理由は`up_to_date`、`range_gap`、`record_budget_exhausted`、 +`round_budget_exhausted`、`retry_exhausted`である。CLIのretry delayは50msから決定的に倍増し1秒でcapする。 +raw exception、descriptor token、timestampはauditへ書かない。 + +registryはimport truthであり、auditはloop/recovery記録である。crashでregistryだけが先に進んだ場合、auditが +最後にpinしたcount位置のregistry record fingerprintをverified registry内でexact照合する。一致する場合だけ +current verified headを`recovered`として記録し、次sequenceから続行する。auditがregistryより先、またはpinした +historical headが異なる場合はfail closedになる。 + +各audit recordのtrust pinは、提示ledgerがそのgenerationを同じtrust fingerprintで含むことを要求する。 +active generationがpinより古ければrollback、同じactive generationでledger fingerprintが違えばforkとして +network接続前に拒否する。contiguous signed extensionだけが`trust_advanced`として受理される。 + +このpinはmode 0600 fileとhash chainによるlocal tamper evidenceであり、registry/audit全体を書き換えられるlocal +attackerへのsecure monotonic counterではない。またloopはprivate keyを読まず、acknowledgementの自動署名・送信は +行わない。次はremote bindを許可する前に、peer Ed25519 identityへbindしたmTLS certificate/SPKI enrollment、 +TLS 1.3-only、address allowlist、plaintext fallback禁止を追加する。 + +### Peer-signed remote mutual TLS + +Phase 55では、既存checkpoint peer trustのEd25519 identityへTLS leaf certificateを署名付きで登録する。 +templateはcertificateの正規化DER SHA-256、SPKI SHA-256、subject/issuer、serial、有効期間、DNS/IP SAN、 +client/server EKUをbindし、秘密鍵を含まない。generation 2以降は直前enrollment fingerprintを必須とする。 + +```bash +morphic benchmark agent-cli-checkpoint-peer-tls-enrollment-template \ + --certificate peer-1-tls.pem \ + --peer-trust checkpoint-peer-trust.json \ + --peer-id peer-1 \ + --generation 1 \ + --output peer-1-tls-template.json \ + --json + +# signing_payload_base64をpeer-1のactive Ed25519 identity keyで外部署名する。 +morphic benchmark agent-cli-checkpoint-peer-tls-enrollment \ + --template peer-1-tls-template.json \ + --certificate peer-1-tls.pem \ + --peer-trust checkpoint-peer-trust.json \ + --key-id peer-1-key-1 \ + --signature-base64 "$PEER_1_TLS_SIGNATURE" \ + --output peer-1-tls-enrollment.json \ + --json + +# tls-enrollments.jsonはenrollments配列に全世代をpeer/generation順不同で格納できる。 +morphic benchmark agent-cli-checkpoint-peer-tls-trust \ + --peer-trust checkpoint-peer-trust.json \ + --enrollments tls-enrollments.json \ + --output checkpoint-peer-tls-trust.json \ + --json +``` + +certificateはCAではないleafで、少なくとも1つのSAN、digital signature key usage、client authとserver authの +両EKUを持つ必要がある。trust生成時は全enrollment signature、peer/trust/registry、連続generation chainを +再検証する。runtimeは各peerの最高generationだけをactiveとし、DERとSPKIの両pinが一致しない旧certificateを +起動前または接続時に拒否する。同じactive certificate/SPKIを複数peerへ割り当てることもできない。 + +```bash +morphic benchmark agent-cli-checkpoint-gossip-mtls-serve \ + --descriptor .morphic/gossip/peer-1-mtls.json \ + --range-bundles signed-checkpoint-ranges.json \ + --cursor-ledger checkpoint-peer-cursors.jsonl \ + --registry-id example-org-checkpoints \ + --source-peer-id peer-1 \ + --peer-trust checkpoint-peer-trust.json \ + --tls-trust checkpoint-peer-tls-trust.json \ + --certificate peer-1-tls.pem \ + --private-key peer-1-tls-key.pem \ + --certificate-authority checkpoint-peer-ca.pem \ + --bind-host 0.0.0.0 \ + --advertised-host 192.0.2.10 \ + --allow-client-address 192.0.2.20 \ + --max-requests 64 \ + --lifetime-seconds 300 + +morphic benchmark agent-cli-checkpoint-gossip-mtls-status \ + --descriptor .morphic/gossip/peer-1-mtls.json \ + --peer-trust checkpoint-peer-trust.json \ + --tls-trust checkpoint-peer-tls-trust.json \ + --client-peer-id peer-2 \ + --certificate peer-2-tls.pem \ + --private-key peer-2-tls-key.pem \ + --certificate-authority checkpoint-peer-ca.pem \ + --server-hostname peer-1.example.org \ + --allow-server-address 192.0.2.10 \ + --json +``` + +transport protocol version 2はTLS 1.3だけを許可し、serverはclient certificateを必須化する。clientはCA chain、 +certificate hostname、descriptor host allowlist、接続後peer address、active server DER/SPKI pinをすべて照合する。 +serverも接続元allowlist、active client DER/SPKI pin、request内client peer IDを照合する。0600 descriptorにtokenや +private keyはなく、平文protocolへのfallbackはない。private key fileはgroup/other accessを許可しない。 + +CA bundleとgenesis peer trustの真正な初回配布、DNS/address運用、certificate revocation status、private key保護は +operator境界である。Phase 55のmTLS CUIはserve/statusまでとし、fetch/ack/syncは次のtransport-neutral統合で扱う。 + +### Mutual-TLS artifact fetch, acknowledgement, and sync + +Phase 56では、artifact操作を特定transportから分離した。`status`、`fetch_range`、 +`submit_acknowledgement`はtyped request senderを受け取り、未指定時だけPhase 53のloopback protocol v1を使う。 +protocol v2のreusable mTLS clientも同じsender contractを実装する。このためtransport追加後もrangeのpeer署名、 +ack response、witness/root/Merkle、registry overlap、sync audit、retry、rollback pinを別実装へ分岐させない。 + +```bash +morphic benchmark agent-cli-checkpoint-gossip-mtls-fetch \ + --descriptor .morphic/gossip/peer-1-mtls.json \ + --start-sequence 0 \ + --max-records 100 \ + --peer-trust-ledger checkpoint-peer-trust-ledger.json \ + --tls-trust checkpoint-peer-tls-trust.json \ + --client-peer-id peer-2 \ + --certificate peer-2-tls.pem \ + --private-key peer-2-tls-key.pem \ + --certificate-authority checkpoint-peer-ca.pem \ + --server-hostname peer-1.example.org \ + --allow-server-address 192.0.2.10 \ + --output received-range.json \ + --json + +morphic benchmark agent-cli-checkpoint-gossip-mtls-ack \ + --descriptor .morphic/gossip/peer-1-mtls.json \ + --acknowledgement signed-acknowledgement.json \ + --peer-trust-ledger checkpoint-peer-trust-ledger.json \ + --tls-trust checkpoint-peer-tls-trust.json \ + --client-peer-id peer-2 \ + --certificate peer-2-tls.pem \ + --private-key peer-2-tls-key.pem \ + --certificate-authority checkpoint-peer-ca.pem \ + --server-hostname peer-1.example.org \ + --allow-server-address 192.0.2.10 \ + --json +``` + +fetchはmTLS responseを受けた後も`SignedCheckpointRangeBundle`をexact peer trust generationで再検証し、 +outputを既存pathへ上書きしない。ackは外部署名済みartifactだけを送信し、返却されたcursor recordが送信ackを +変更していないことを確認する。TLS transportはack用peer private keyを読まず、client TLS keyだけをhandshakeに使う。 + +bounded catch-upも同じmTLS senderを注入できる。 + +```bash +morphic benchmark agent-cli-checkpoint-gossip-mtls-sync \ + --descriptor .morphic/gossip/peer-1-mtls.json \ + --registry checkpoint-registry.jsonl \ + --sync-audit checkpoint-sync-peer-1.jsonl \ + --registry-id example-org-checkpoints \ + --source-peer-id peer-1 \ + --peer-trust-ledger checkpoint-peer-trust-ledger.json \ + --witness-trust witness-trust.json \ + --authority-root-ledger authority-root-ledger.json \ + --tls-trust checkpoint-peer-tls-trust.json \ + --client-peer-id peer-2 \ + --certificate peer-2-tls.pem \ + --private-key peer-2-tls-key.pem \ + --certificate-authority checkpoint-peer-ca.pem \ + --server-hostname peer-1.example.org \ + --allow-server-address 192.0.2.10 \ + --max-rounds 16 \ + --max-records 1000 \ + --max-attempts 3 \ + --json +``` + +mTLS trustは提示peer-trust ledgerのactive generationに対して全enrollment署名を再検証してから接続する。 +sync loopはPhase 54と同じwhole-loop lock、hash-chained audit、crash recovery、trust rollback/fork pin、 +round/record/attempt budget、machine-readable stop reasonを使う。transport errorだけが既存bounded retry対象になる。 + +certificateの期限はTLS handshakeで検証されるが、接続前の運用警告、署名付き明示revocation、grace period、 +OCSP/CRL policyはまだない。authenticated peer discovery、automatic enrollment/ack signing、daemon化も次の境界である。 + +--- + ## Agent CLI Router ```python diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9a64bbe..2b0167e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -432,7 +432,7 @@ morphic-agent/ │ ├── marketplace.py # morphic marketplace {search|install|list|suggest|uninstall} (Sprint 5.3) │ ├── evolution.py # morphic evolution {stats|failures|update|report} (Sprint 6.1) │ ├── cognitive.py # morphic cognitive {state|delete|affinity|handoff|insights} (Sprint 7.5) -│ └── benchmark.py # morphic benchmark {run|continuity|dedup} (Sprint 7.6) +│ └── benchmark.py # UCL + recorded agent CLI comparison commands │ ├── shared/ │ └── config.py # pydantic-settings (all env vars + marketplace + evolution settings) @@ -554,8 +554,28 @@ morphic-agent/ │ ├── test_agent_engines.py # 37 tests (6 engines live + routing + fallback + availability, Sprint 4.4-4.5) │ └── test_ucl_cross_engine.py # 13 tests (handoff pipeline + adapter fidelity + insight roundtrip + affinity + conflict + benchmarks, Sprint 7.6) │ -├── benchmarks/ # Sprint 7.6: Context continuity + dedup accuracy benchmarks +├── benchmarks/ # Deterministic and opt-in live benchmark utilities │ ├── __init__.py +│ ├── agent_cli_adjudication.py # Evidence/review join into final observations +│ ├── agent_cli_attestation.py # Offline Ed25519 reviewer provenance verification +│ ├── agent_cli_authority.py # Organization enrollment + signed campaign envelope +│ ├── agent_cli_transparency.py # Root rotation ledger + Merkle inclusion proofs +│ ├── agent_cli_witness.py # Compact consistency + quorum checkpoints +│ ├── agent_cli_checkpoint_registry.py # Registry + signed peer range/cursor sync +│ ├── agent_cli_peer_trust_ledger.py # Majority-signed peer key rollover continuity +│ ├── agent_cli_gossip_transport.py # Bounded challenge-authenticated loopback protocol +│ ├── agent_cli_gossip.py # Transport-neutral signed range/ack operations + service +│ ├── agent_cli_gossip_sync.py # Sender-injected trust-pinned pull loop + durable audit +│ ├── agent_cli_gossip_tls_identity.py # Peer-signed leaf/SPKI enrollment + rotation pins +│ ├── agent_cli_gossip_tls_transport.py # TLS 1.3 mutual-auth remote gossip protocol +│ ├── agent_cli_campaign.py # Read-only lifecycle status across artifacts +│ ├── agent_cli_comparison.py # Recorded same-task 3-arm validation and metrics +│ ├── agent_cli_preflight.py # Non-authorizing campaign and review bindings +│ ├── agent_cli_receipts.py # Codex/Claude/Morphic normalized cost receipts +│ ├── agent_cli_recorder.py # Opt-in detached-worktree trial evidence recorder +│ ├── agent_cli_review_policy.py # Declared operator/reviewer separation rules +│ ├── agent_cli_rehearsal.py # Internal zero-cost end-to-end recorder rehearsal +│ ├── templates/ # Parseable agent CLI manifest/recorder examples │ ├── context_continuity.py # AdapterScore, ContinuityResult, run_benchmark() — target >85% │ ├── dedup_accuracy.py # DedupScore, DedupResult, run_benchmark() — target >50% │ └── runner.py # BenchmarkSuiteResult, run_all() — unified benchmark runner diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cb79dcd..be83434 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,56 @@ # CLAUDE.md Changelog ## Unreleased - +- **[FEAT/BENCHMARK]** mTLS status CUIへ`tls_expiry_warnings`を追加し、期限警告をJSON・human outputから観測可能にした。 +- **[TEST/BENCHMARK]** Phase 61 TLS revocation CLIのtemplate/finalization round-trip回帰fixtureを追加。 +- **[FEAT/BENCHMARK]** Phase 60 TLS revocation template/finalization CUIを追加。 +- **[FEAT/BENCHMARK]** Phase 59 TLS revocation issuance APIを追加: private-key-free templateが対象generation・reason・時刻・trust hashとeligible active keyをbindし、finalizationでEd25519署名を検証してrevocation artifactを発行する。 + +- **[FEAT/BENCHMARK]** Phase 58 TLS trust CLIへ`--revocations`を追加し、peer-signed revocation bundleを検証したrevocation-aware trust artifactを発行可能にした。 + +- **[FEAT/BENCHMARK]** Phase 57 TLS revocation/expiry policyを追加: active peer Ed25519 keyで署名されたcertificate enrollment revocation chainをTLS trust fingerprintへbindし、revoked generationをactive pin解決から除外。mTLS server/clientはtransport開始前にexpired leafを拒否し、設定可能なpre-expiry warning windowを公開する。 + +- **[FEAT/BENCHMARK]** Phase 56 mutual-TLS artifact syncをTDDで追加: checkpoint gossipのstatus/fetch/ackをtyped authenticated request senderへ分離し、protocol-v1 loopbackをdefaultとして完全互換のまま維持。reusable protocol-v2 mTLS clientを同じcontractへ接続し、range受信後のpeer Ed25519署名・exact trust generation検証、ack response同一性検証をtransport外の既存pathで継続する。Phase 54 bounded resumable syncもsender注入へ対応し、whole-loop lock、hash-chained audit、crash recovery、trust rollback/fork pin、retry/budget/stop reasonを変更せずmTLSで利用可能にした。active peer-trust generationに対してTLS enrollment署名を再検証するmTLS fetch/ack/sync CLIを追加。明示certificate revocation/expiry warning、discovery、automatic signingは未保証。 +- **[FEAT/BENCHMARK]** Phase 55 peer-signed remote mutual TLSをTDDで追加: TLS leafの正規化DER/SPKI SHA-256、subject/issuer、serial、有効期間、SAN、client/server EKUを既存peer trustのactive Ed25519 identity署名へbindし、per-peer contiguous generation chainとactive pinをself-fingerprinted TLS trustとして発行する。署名・registry/trust・predecessor・cross-peer active pin重複を再検証し、旧certificateを拒否。protocol v2はTLS 1.3-onlyかつclient certificate必須で、CA/hostname、明示client/server IP allowlist、接続後address、request peer ID、active leaf/SPKI pinを相互照合する。token-free 0600 descriptor、one-use nonce replay防止、64KiB/2MiB、concurrency/request/timeout上限、deterministic cleanupを維持し、plaintext fallbackを禁止。private-key-free enrollment template/finalization/trustとmTLS serve/status CLIを追加。CA/genesis配布、revocation status、mTLS fetch/ack/sync CLI、discovery、automatic signingは未保証。 +- **[FEAT/BENCHMARK]** Phase 54 durable bounded gossip catch-up loopをTDDで追加: authenticated statusから次のlocal sequenceを含むpre-signed rangeを選び、client側peer署名と既存authority/witness/Merkle/registry検証を通してmissing suffixだけをatomic importする。mode 0600 sync auditはwhole-loop `flock(LOCK_EX|LOCK_NB)`で多重実行を拒否し、`imported/retry/recovered/trust_advanced/stopped`をtimestamp・secret・raw exceptionなしのself-fingerprinted JSONL hash chainへfsync。verified registryがauditより先行するcrash状態はexact historical head照合後に`recovered`として再開する。auditは最高peer-trust generation/trust/ledger fingerprintをpinし、古いledger rollbackとpin済みgeneration forkをnetwork接続前に拒否し、正当なcontiguous extensionだけを受理。round/record/attempt/backoffをboundし、`up_to_date/range_gap/record_budget_exhausted/round_budget_exhausted/retry_exhausted`をmachine-readableに返すprivate-key-free gossip-sync CLIを追加。local audit全書換え耐性、automatic acknowledgement signing、remote mTLSは未保証。 +- **[FEAT/BENCHMARK]** Phase 53 authenticated loopback checkpoint gossipをTDDで追加: 明示起動時だけ`127.0.0.1`のrandom portへbindし、0700 directory / 0600 descriptorへprotocol version、registry、source peer、instance、32-byte bearer tokenを記録する。各接続はone-use 32-byte client nonceとserver nonceのchallengeを経て、protocol/instance/registry/source/operation/payloadをHMAC-SHA256へbindし、responseも認証。nonce replay、token/endpoint/version不一致、64KiB request超過、不正artifactをfail closedで拒否し、2MiB response、8 concurrent clients、1,024 nonce/request cap、2秒timeout、listener lifetimeを固定した。serverは事前署名済みexact rangeを起動前に検証して配布し、clientはtrust snapshot/ledgerで再検証。signed acknowledgementはdurable monotonic cursor storeを通す。serve/status/fetch/ack CLIを追加し、max request到達・lifetime・cancel時にlistener/client task/writerを閉じowned descriptorだけを削除。remote/TLS、discovery、automatic signing/catch-up loopは未保証。 +- **[FEAT/BENCHMARK]** Phase 52 peer-trust rollover continuityをTDDで追加: peer trust generation 2以降を直前trustのactive peer strict-majority Ed25519署名で承認する。rotation statementはregistry、generation、predecessor/successor trust fingerprint、自動計算quorumをbindし、active predecessor peerごとのprivate-key-free signing requestを発行。minority、duplicate peer、successor-only/revoked key、不正署名、generation gap/reorder、registry変更、trust reuse、certificate/ledger改ざんをfail closedで拒否する。out-of-band genesisからcontiguous self-fingerprinted ledgerを構築し、historical trust resolverをcursor replay/storeへ統合したため、rotation前後のacknowledgementを同一cursor ledgerで検証・前進できる。rotation-template、trust-ledger、`--peer-trust-ledger` cursor CLIを追加。genesis配布・ledger rollback防止・online transportは未保証。 +- **[FEAT/BENCHMARK]** Phase 51 authenticated range sync + durable peer cursorsをTDDで追加: bounded contiguous checkpoint record rangeのbase previous hash、first/last sequenceとrecord hash、全record fingerprint列、registry/peer trustを1つのEd25519署名へbindする。importはpeer署名と全recordのauthority/witness/Merkle/chain検証を先行し、exclusive lock内でlocal overlapをexact照合して不足suffixだけを追記。gap/fork/invalid rangeは0 byte mutation、write failureは元sizeへtruncateしてprocess-level atomicityを保つ。受信peerのackはexact signed rangeとapplied head/root/sizeを署名し、送信側はmode 0600・locked・hash-chained cursor JSONLへpeer組ごとの単調positionとして保存する。retryは冪等、regression、same-sequence conflict、signature/fingerprint/chain tamperingをfail closedで拒否。private keyを読まないrange export/import、ack template、cursor store/status CLIを追加。online transportとpeer-trust rotation ledgerは未保証。 +- **[FEAT/BENCHMARK]** Phase 50 durable checkpoint registry + authenticated peer exchangeをTDDで追加: witnessed consistency proof/checkpointをdeterministic self-fingerprinted recordへ格納し、sequenceとprevious record SHA-256でmode 0600 JSONL hash chainを形成する。全replayでroot ledger、witness trust、Merkle consistency、authority/witness signatures、record fingerprint、連続性を再検証。appendは`flock` + `O_APPEND` + `fsync`で直列化し、同一retryは冪等、stale extension、gap、改ざん、truncated tail、same-size different-root split viewをfail closedで拒否する。self-fingerprinted peer Ed25519 trustと署名packetはsource peer/key、exact registry record/checkpoint/root/sizeをbindし、peer認証後だけ同じatomic append経路へimportする。private keyを読まないpeer-trust/status/store/export-template/import CLIとexampleを追加。online listener、peer discovery、real-world identity attestation、multi-record range syncは未保証。 +- **[FEAT/BENCHMARK]** Phase 49 compact consistency + quorum witness checkpointsをTDDで追加: RFC 6962 `SUBPROOF` recursionで`0 < previous size < current size`のunique minimal Merkle consistency pathを生成し、complete logなしで旧/new signed rootを同時再構成してappend-only growthを検証する。3→7 RFC example shapeと1〜11→最大12 leafの全組合せを検証。self-fingerprinted Ed25519 witness trustは全witnessのactive keyとstrict-majority quorumを必須化し、exact log/root ledger/old+new tree heads/consistency proofへdetached witness signaturesをbindする。quorum不足、unknown/revoked key、不正署名、same-size different-root split viewをfail closedで拒否。opt-in witness-bound statusは包含後も`witness_pending`に留まり、consistency proofとwitness quorum検証後だけ`finalized`になる。private keyを読まないconsistency/witness trust/checkpoint template CLIとexampleを追加し、Phase 48 inclusion-onlyとlegacy pathを維持。witness間のnetwork gossip、key identity attestation、durable checkpoint exchangeは未保証。 +- **[FEAT/BENCHMARK]** Phase 48 authority-root continuity + append-only transparencyをTDDで追加: predecessor Ed25519 signatureで連結したcontiguous root generations、active-root署名付きself-fingerprinted ledger、既知rootのrevocationとactive-root revocation拒否を実装。reviewer trust、finalization、campaign envelopeをexact ledgerへbindし、RFC 6962形式の`SHA256(0x00 || leaf)` / `SHA256(0x01 || left || right)` Merkle root、signed tree head、inclusion proof、complete-log prefixによるappend-only extension検証を追加した。ledger-bound statusはroot不足時`authority_root_pending`、envelope未署名時`campaign_envelope_pending`、包含証明不足時`transparency_pending`に留まり、全検証後だけ`finalized`になる。private keyを読まないrotation/ledger/tree-head templateとoffline log/proof CLIを追加し、Phase 47 authority-boundとunsigned legacy artifactのhash/signature bytesを維持。root初回配布・compromise時のout-of-band trust reset・compact consistency proof・gossip・OIDC/Sigstore identityは未保証。 +- **[FEAT/BENCHMARK]** Phase 47 organization-authority anchored campaign provenanceをTDDで追加: self-fingerprinted offline Ed25519 authorityをreviewer trustへbindし、trust内の全reviewer keyへauthority-signed enrollment certificateをexactly once要求。秘密鍵を読まない`agent-cli-reviewer-enrollment-template`と、manifest/preflight/evidence/reviews/policy/trust/enrollments/attestations/resultsを1つのnon-authorizing payloadへbindする`agent-cli-campaign-envelope-template`を追加した。authority-bound finalize/statusはmissing/mixed/invalid enrollmentを拒否し、最終authority signatureがない間は`campaign_envelope_pending`、enrollment不足時は`reviewer_enrollment_pending`に留まる。unanchored Phase 46とunsigned legacy pathは維持。root key配布、expiry、root rotation/revocation、transparency logは未保証として明示する。 +- **[FEAT/BENCHMARK]** Phase 46 signed reviewer provenanceをTDDで追加: review policyへ結び付くself-fingerprinted Ed25519 public-key trust declaration、active/revoked key rotation、completed review全体とreviewer別decision集合をbindするcanonical signing payload、distinct reviewer全員のdetached signature検証を実装。Morphicは秘密鍵を読まず、unknown/revoked key、invalid signature、欠落reviewer、別reviews/policy/trust artifact混入をfail closedで拒否する。`agent-cli-attestation-template`は署名対象だけをexclusive出力し、trust-bound finalize/statusはpolicy・trust・attestation bundleを必須化。statusへ`review_attestation_pending`を加える一方、unsigned legacy campaignは後方互換で維持する。署名は登録鍵の保有証明であり、trust enrollmentを実在人物の本人確認とは表現しない。 +- **[FEAT/BENCHMARK]** Phase 45 reviewer separation + read-only campaign statusをTDDで追加: self-fingerprinted review policyがoperator ID、allowed reviewer IDs、minimum distinct reviewer countを固定し、template/completed reviewへbind。operator self-review、allowlist外ID、distinct不足、decision matrixでは実現不能なpolicyを拒否する一方、宣言IDを本人認証とは表現しない。`morphic benchmark agent-cli-status`はmanifest/preflight/evidence/pending-or-completed review/resultsの順序とidentity/hash/estimate/policy/finalized resultを副作用なしで再検証し、6段階のstageとnext actionをdeterministic JSONで返す。すべてのstageで`paid_execution_authorized=false`を固定する。 +- **[FEAT/BENCHMARK]** Phase 44 non-authorizing campaign preflight + evidence-bound review templateをTDDで追加: `morphic benchmark agent-cli-preflight`がmanifestのfull resolved Git commit、exact 3-arm runtime declarations、recorder coverageをoffline検証し、normalized runtime versions、完全なmanifest/config、arm/check/handoff commandsをSHA-256化してraw goalを公開せず契約全体をbind。artifact fingerprintと`execution_authorized=false`を固定し、version command・agent・paid APIは起動しない。`agent-cli-review-template`は全arm/trialのhuman fieldsをnull、`review_completed=false`としてexact preflight/evidence/agent argv fingerprintsへbind。reviewerが全項目を埋め`review_completed=true`へ変更後、`agent-cli-finalize --preflight`がbindingを検証する。別evidenceへのreview流用を拒否しつつ、binding fieldなしのPhase 42 reviewは後方互換で受理する。 +- **[FEAT/BENCHMARK]** Phase 43 first-party Morphic receipt + zero-cost rehearsalをTDDで追加: `morphic code --benchmark-receipt`が通常response後のfinal stdout lineへcanonical `morphic_benchmark_receipt`を出力し、council turn costとnormalized completion usageだけを集約。失敗/Ctrl-Cは未確定費用を0ドルと偽らずreceipt欠損としてfail closedする。`morphic benchmark agent-cli-rehearse`はexternal agent/APIを設定・起動できない内部Python fixtureだけでpinned detached-worktree 3-arm matrix、Codex/Claude/Morphic receipt parse、hashed evidence、review fingerprint、offline finalizeをactual/configured $0で完走し、manifest/config/evidence/reviews/resultsを既存directory非上書きで発行する。parseable dry-run templatesを追加し、synthetic reviewはaccepted patchを常にfalseとして品質証拠と混同しない。 +- **[FEAT/BENCHMARK]** Phase 42 provider receipt normalization + deterministic adjudicationをTDDで追加: recorderがraw stdout破棄前にCodex JSONL usage、Claude stream-json provider cost、Morphic canonical receipt envelopeをprivacy-safe receiptへ正規化。Codex costはmodel/usageから再計算してtamperingを検出し、provider別cost source、non-negative usage、zero parse errorsを強制する。`morphic benchmark agent-cli-finalize`がcomplete evidence/review matrixをtask/revision/provider/argv fingerprintで結合し、accepted patch・human intervention・recoveryとmachine-derived check/handoffをPhase 40 observationsへ変換。receipt欠損、duplicate/mismatch、failed run acceptance、authorized cap超過、既存output上書きを拒否し、agent/paid APIを起動せずdeterministic JSONを生成する。 +- **[FEAT/BENCHMARK]** Phase 41 explicit opt-in isolated trial recorderをTDDで追加: `morphic benchmark agent-cli-record`はdefaultでfilesystem/processを変更しないdeterministic planを返し、live executionには`--execute` + `--acknowledge-paid` + 全trialのconfigured maximum estimateを覆う`--cost-cap-usd`を必須化。Codex/Claude/Morphicの各arm/trialをpinned revisionのsource外detached worktreeへ隔離し、argvをshellなしで実行、per-command timeoutとfinally cleanupを適用する。evidenceはargv/stdout/stderrのSHA-256・byte数・exit/timeout/elapsed・check/handoff結果だけを保存しraw prompt/outputを残さず、既存evidenceを上書きしない。actual provider cost/accepted patchは`pending_adjudication`のまま保持し、estimate authorizationをhard billing capとは表現しない。 +- **[FEAT/BENCHMARK]** Phase 40 recorded same-task agent CLI comparisonをTDDで追加: one manifestで同一task/workspace revision/checks/handoff assertions/repetitionsを固定し、`codex_cli` / `claude_code` / `morphic_control`のcomplete trial matrixを検証。completion、accepted patch、verification、median elapsed、cost、human interventions、recovery、context handoffをarm別に算出し、恣意的なcomposite scoreなしでmetric leadersを出力する。duplicate/missing/mismatched/undeclared observationは拒否し、`morphic benchmark agent-cli --manifest ... --results ... [--json]`はtimestamp-free/sorted-key deterministic reportを生成。offline evaluatorはnative engine/paid APIを起動せず、`benchmarks` packageをwheelへ同梱する。 +- **[FEAT/CHAT-CLI]** Phase 39 provider-neutral steeringをTDDで追加: authenticated `morphic chat-control steer `がnon-empty/max 2048 UTF-8 bytesのreplacementをfirst-writer-winsでqueueし、active turnをcancel。provider cleanup後にledger replayでnative session/workspace/permission provenanceを復元し、`turn_steered` audit eventの後へ通常の`user_message`として同じCodex/Claude sessionへ再投入する。cleanup中の上書き、invalid/oversized promptを拒否し、slash-prefixed remote promptもlocal commandとして解釈しない。 +- **[FEAT/CHAT-CLI]** Phase 38 authenticated loopback controlをTDDで追加: explicit `morphic chat --control`時のみactive turnの期間に限って`127.0.0.1` random port serverを起動し、protocol version/session/port/random tokenをmode 0600 descriptorへ保存（control directoryは0700）。`morphic chat-control status/cancel`から別terminalで操作でき、token/session/command mismatchとnon-loopback descriptorはfail closed。turn完了またはcancellation cleanup後はowned descriptorを削除し、permanent listenerやprovider process detailsを公開しない。 +- **[FEAT/CHAT-CLI]** Phase 37 active-turn controlをTDDで追加: injectable `ActiveTurnController`がinteractive chatの1 turnをchild taskとして所有し、active turn中のCtrl-Cだけをchildへ配送してprovider cleanupと`turn_cancelled`永続化を完了後、REPLを終了せず次promptへ戻す。中断後はledger replayでsequence/native session stateを再構築し、partial eventsとのsequence衝突を防止。non-streaming runtimeもuser/cancellationを同じ台帳契約で保存する。outer task cancellationは`CancelledError`のまま透過し、idle時とone-shot codeのCtrl-C exit 130も維持する。 +- **[FIX/CHAT-CLI]** Phase 36 durable turn cancellationをTDDで追加: streaming turnのcaller cancellationを`turn_cancelled`として、それまでに到着済みのnative engine eventsの後へappend-only ledger記録する。記録後はoriginal `CancelledError`を再送出し、`morphic chat` / `morphic code`はCtrl-Cを`Cancelled.`とexit code 130で一貫して終了する。cancelled turnをassistant successとして生成しない。 +- **[FIX/AGENT-CLI]** Phase 35 provider-pinned native resumeをTDDで追加: route resume requestへowner `resume_engine`を必須化し、preferred engineとの不一致を拒否。fallback chain内のnon-owner enginesはavailability check/subprocess起動前に`resume_engine_mismatch`でskipする。これによりClaude session idをCodexへ、Codex thread idをClaudeへ渡すcross-provider fallbackをfail closed。 +- **[FEAT/AGENT-CLI]** Phase 34 Claude native streaming/resumeをTDDで追加: Claude `stream-json`のsystem init、assistant text/tool_use、user tool_result、final resultをprovider-independent engine eventsへ正規化しraw payloadを保持。Claude driverが`ResumableStreamingScopedAgentEnginePort`を実装し、explicit session idを同一workspace/permission provenance下で`--resume`する。Claude-reported output/model/usage/total costをresultへ保持し、Chat CLI `--route-direct --engine claude_code`を開放。 +- **[FEAT/AGENT-CLI]** Phase 33 scoped Claude Code adapterをTDDで追加: subprocess runnerへexplicit `cwd`を追加し、Claude driverが`ScopedAgentEnginePort`を実装。Morphic `read-only`→Claude `plan`、`workspace-write`→`acceptEdits`、`danger-full-access`→explicit `bypassPermissions` + dangerous flagへmappingし、headlessでpromptを保持できない`confirm-destructive`は拒否。従来の`--setting-sources user`とhard-coded tool allowlistを撤廃し、project/local settings、CLAUDE.md、skills、hooks、MCP、pluginsのnative harness behaviorを保持する。 +- **[FIX/CHAT-CLI]** Phase 32 native subprocess cancellation cleanupをTDDで追加: buffered/streaming CLI taskがcancelされた場合、child processへ`terminate`を送り、2秒のgraceを超えた場合のみ`kill`へ昇格する。cleanup後もoriginal `CancelledError`をcallerへ再送出するため、Ctrl-C semanticsを維持しつつorphan Codex processを防止。 +- **[FEAT/CHAT-CLI]** Phase 31 scoped Codex thread resumeをTDDで追加: `ChatSession`がnative engine/thread idをoriginal workspace rootとpermission modeにbindし、append-only ledger replayでresume時に復元する。`ResumableStreamingScopedAgentEnginePort`を分離し、Codexはexplicit `--sandbox` / `--cd`を維持した`codex exec ... resume `を実行。同じMorphic sessionの次turnはstored Codex threadを継続し、workspace/permission mismatchまたはstream/scope context欠落時はroute実行前にfail closedする。global `--last`は使用しない。 +- **[FEAT/CHAT-CLI]** Phase 30 live native progress renderingをTDDで追加: `SendChatMessageUseCase`にoptional event observer fan-outを追加し、durable ledger append成功後にのみpresentationへ通知。Chat REPL / one-shot codeのstreaming direct pathで`NativeEventProgressRenderer`を使い、run/tool/file/plan/completion/errorのallowlistだけを短く逐次表示する。raw provider payload、hidden reasoning、assistant message、unknown eventsは表示せず、whitespace compact + 160文字上限を適用。observer failureはaudit stateやnative executionを壊さないbest-effort扱い。 +- **[FEAT/CHAT-CLI]** Phase 29 incremental Codex event streamingをTDDで追加: async subprocess runnerがstdout/stderrをconcurrentにdrainし、Codex JSONL stdoutをprocess完了前にline単位でstateful decodeして`AgentEngineEventSinkPort`へpublishする。`StreamingScopedAgentEnginePort` / `StreamingCouncilRuntimePort`でcapabilityを明示し、route-directではuser messageを先に永続化してnative eventsを到着時にappend。final buffered metadataは非streaming互換用に保持しつつ、streaming chat ledgerへの二重記録を抑止。実subprocess line deliveryを含むunit testsを追加。 +- **[FEAT/CHAT-CLI]** Phase 28 native event ledger/scoped execution contractをTDDで追加: direct native resultのnormalized eventsを各`CouncilTurn`へ付与し、`SendChatMessageUseCase`がprovider順の独立`engine_event`としてcouncil argument/decision/assistant responseより前にappend-only session ledgerへ保存する。workspace rootとpermission modeを扱う契約を`ScopedAgentEnginePort`へ分離し、既存`AgentEnginePort`互換性を維持。scoped execution要求時は非対応adapterへ権限制御を黙って落とさずskipし、Codexなど明示対応adapterへfallbackする。 +- **[FEAT/CHAT-CLI]** Phase 27 Codex permission/JSONL normalizationをTDDで追加: provider-independent `AgentEngineEvent` vocabularyを追加し、`codex exec --json` の JSONL (`thread.started`, `turn.*`, `item.*`, `error`) をrun/tool/file/plan/assistant/completion/failure eventsへ正規化。thread id、final assistant message、usage、raw payload、parse error countを `AgentEngineResult.metadata` に保持する。Codex driverはdeprecated `--full-auto`を廃止し、Morphic `read-only` / `workspace-write` / `danger-full-access` をexplicit `--sandbox`へ、workspace rootを`--cd`へ伝播。non-interactive modeでapproval promptを保持できない`confirm-destructive`は拒否し、他native adaptersのpermission mapping完了まではdirect routeをexplicit `--engine codex_cli`に限定。 +- **[FEAT/CHAT-CLI]** Phase 26 single-engine direct routeをTDDで追加: `RouteChatDirectRuntime` が既存 `RouteToEngineUseCase` を1 turnにつきexactly once呼び、resultを単一`IMPLEMENTER` turn/decisionへ正規化。`morphic chat` / `morphic code` に `--route-direct` とoptional `--engine` を追加し、`--route-council`との同時指定を拒否。route failure/empty outputはlocal deterministic successへ隠さずuser-facing error + non-zero exitにする。現行external CLI driversはMorphic permission modeをnative権限へ未伝播のため、direct executionは当面explicit `danger-full-access`を必須とする。 +- **[FEAT/CHAT-CLI]** Phase 25 CLI permission mode controlsをTDDで追加: `morphic chat` / `morphic code` に `--permission-mode` を追加し、既存 `PermissionMode` (`read-only`, `workspace-write`, `confirm-destructive`, `danger-full-access`) をsession start ledgerへ保存できるようにした。`/status` は選択modeを表示し、`read-only` sessionでmutating `/tools run` が発生した場合はREPLをクラッシュさせず `permission denied: ...` をassistant messageとして記録する。 +- **[FEAT/CHAT-CLI]** Phase 24 REPL tool run failure reportingをTDDで追加: `/tools run` が LAEE denied/error result を `success=False exit_code=... error=...` として表示するようにし、失敗理由がREPL上で見えるようにした。REPLは `RiskAssessor` でtool name + JSON argumentsからriskを評価してから `ExecuteChatToolUseCase` へ渡す。`MORPHIC_CHAT_TOOL_EXECUTION=laee` opt-in時の `/tools run fs_delete ...` がconfirm-destructiveでdenyされ、対象fileを残し、audit logへ記録されることを検証。 +- **[FEAT/CHAT-CLI]** Phase 23 REPL tool run LAEE opt-inをTDDで追加: Chat CLI tool executor factoryを追加し、defaultはsafeな `NoopToolExecutor`、`MORPHIC_CHAT_TOOL_EXECUTION=laee` の明示opt-in時のみ `LaeeToolExecutor` を選択するようにした。LAEE modeはshared local executor settingsを使い、`morphic chat --doctor --json` に `tool_execution_mode` を出力。REPL `/tools run shell_exec ...` がopt-in時にLAEE audit logへ記録されることを検証。 +- **[FEAT/CHAT-CLI]** Phase 22 REPL tool run no-op UXをTDDで追加: safe defaultの `NoopToolExecutor` を追加し、`morphic chat` 内で `/tools run [json_arguments]` を実行できるようにした。tool runは `ExecuteChatToolUseCase` を通り、既存hook runner flowでpre/post hooksを記録しつつ、defaultでは実toolを起動せずsession ledgerへ `tool_call_requested` / `tool_call_completed` を記録。invalid JSON argumentsはuser-facing messageを返す。 +- **[FEAT/CHAT-CLI]** Phase 21 REPL hook run UXをTDDで追加: `morphic chat` 内で `/hooks run ` を実行できるようにし、slash command eventとhook execution eventsを同一chat session ledgerへ記録。defaultはno-op、`MORPHIC_CHAT_HOOK_EXECUTION=shell` opt-in時はLAEE `shell_exec` 経由で実行しaudit logへ記録されることを検証。これによりDeferred `D008 Hook command execution` を完了扱いに更新。 +- **[FEAT/CHAT-CLI]** Phase 20 manual hook run CLIをTDDで追加: `morphic hooks run ` を追加し、hook execution eventsを `.morphic/sessions/*.jsonl` に永続化。`--json` はsession id / hook execution mode / diagnostics / events / results / summaryを返す。defaultはsafeなno-opのまま維持し、`MORPHIC_CHAT_HOOK_EXECUTION=shell` opt-in時はLAEE `shell_exec` 経由で実行しaudit logへ記録されることを検証。 +- **[FEAT/CHAT-CLI]** Phase 19 hook execution mode wiringをTDDで追加: Chat CLI hook executor factoryを追加し、defaultはsafeな `NoopHookExecutor`、`MORPHIC_CHAT_HOOK_EXECUTION=shell` の明示opt-in時のみ `ShellHookExecutor` を選択するようにした。shell modeはLAEE local executor settingsからapproval/audit/undo設定を引き継ぐ。未知modeはvalidation errorにし、`morphic chat --doctor --json` に `hook_execution_mode` を出力。 +- **[FEAT/CHAT-CLI]** Phase 18 shell-backed hook executorをTDDで追加: `ShellHookExecutor` が hook command を LAEE `LocalExecutorPort` の `shell_exec` actionへ変換し、workspace root `cwd` と timeout を付与して実行する。LAEE successはhook successへ、DENIED/ERRORはfailed hook resultへ正規化。`ExecuteChatToolUseCase` は injected `pre_tool` hook runner がfailed resultを返した場合、tool本体を実行せず停止する。post-tool hook failureはrollbackせずledger dataとして残す。 +- **[FEAT/CHAT-CLI]** Phase 17 hook runner wiringをTDDで追加: safe wiring用の `NoopHookExecutor` adapterを追加し、`ExecuteChatToolUseCase` がoptional `ExecuteChatHookUseCase` を受け取れるようにした。hook runner注入時は `pre_tool` hook execution eventsをtool execution前に、`post_tool` hook execution eventsをexecution後にsession ledgerへ記録。既存のhook planner注入時のplanning挙動は維持し、real shell-backed hook executionは引き続きdeferred。 +- **[FEAT/CHAT-CLI]** Phase 16 hook execution use caseをTDDで追加: `HookExecutionRequest` / `HookExecutionResult` と `HookExecutorPort` を追加し、`ExecuteChatHookUseCase` が enabled hook をport越しに実行して `hook_execution_requested` / `hook_execution_completed` eventsをsession ledgerへ記録。disabled hookはexecutorを呼ばず `hook_execution_skipped` として記録し、FAIL diagnosticsがある場合は実行を拒否。shell-backed hook executorとapproval/risk wiringは引き続きdeferred。 - **[FEAT/CHAT-CLI]** Phase 15 hook planningをtool harnessへ接続: `ExecuteChatToolUseCase` がoptional `PlanChatHooksUseCase` を受け取り、`pre_tool` hook plan eventsをtool execution前に、`post_tool` hook plan eventsをexecution後にsession ledgerへ記録。hook planner未注入時の既存tool execution挙動は維持し、actual hook command executionは引き続きdeferred。 - **[FEAT/CHAT-CLI]** Phase 14 hook execution planningをTDDで追加: `.morphic/hooks/*.json` のvalidated metadataを `HookRegistryPort` 経由で取得し、`PlanChatHooksUseCase` が `hook_execution_planned` / `hook_execution_skipped` eventsをsession ledgerへ記録。FAIL diagnosticsがある場合はplanningを拒否し、actual shell command executionは未実装のままdeferred。 - **[FEAT/CHAT-CLI]** Phase 13 hook diagnosticsをTDDで追加: `.morphic/hooks/*.json` を実行せずread-only validationし、hook type / command / enabled flag / secret-path risk を診断する `morphic doctor hooks` と `morphic doctor hooks --json` を追加。Invalid hooksはisolated reportされ、FAILのみexit 1、WARNはexit 0。 diff --git a/docs/CONTINUATION.md b/docs/CONTINUATION.md index c671da7..d2e8146 100644 --- a/docs/CONTINUATION.md +++ b/docs/CONTINUATION.md @@ -1,7 +1,35 @@ # Morphic-Agent — Continuation State -> Last updated: 2026-06-26 -> Latest work: Morphic Chat CLI Phase 15 hook planning in tool harness +## Final smoke checkpoint (2026-07-25) + +- mTLS status exposes deterministic TLS expiry warnings. + +## Phase 61 checkpoint (2026-07-25) + +- Added end-to-end revocation CLI regression fixture from trust load through finalized artifact. + +## Phase 60 checkpoint (2026-07-25) + +- Added offline revocation template and signature-finalization CLI commands. + +## Phase 57 checkpoint (2026-07-23) + +- Peer-signed TLS revocation chain and trust-bound revoked-generation filtering implemented. +- mTLS server/client expiry rejection plus deterministic warning-window reporting implemented. +- Unit suite: 3,700 passed. CUI issuance/trust commands and dedicated revocation fixtures remain next. + +## Phase 58 checkpoint (2026-07-24) + +- TLS trust CUI accepts optional signed `--revocations` bundle and includes it in the published trust. +- Focused TLS tests and Ruff pass; next slice is dedicated revocation fixture coverage and issuance templates. + +## Phase 59 checkpoint (2026-07-25) + +- Added revocation template/finalization APIs with active-key eligibility and detached signature verification. +- Focused TLS tests and Ruff pass; next slice is CLI commands and dedicated revocation regression fixtures. + +> Last updated: 2026-07-22 +> Latest work: Morphic Chat CLI Phase 56 mutual-TLS artifact sync ## Latest Session Notes (2026-06-26) @@ -102,6 +130,382 @@ Phase 15 implemented: - Existing tool execution behavior is unchanged when no hook planner is injected. - Session ledger ordering is preserved across hook plan, diff, tool requested/completed, verification, and post hook plan events. +Phase 16 implemented: +- Added hook execution domain contracts: `HookExecutionRequest`, `HookExecutionResult`, and `HookExecutorPort`. +- Added `hook_execution_requested` and `hook_execution_completed` chat session events. +- Added `ExecuteChatHookUseCase`, which validates hook diagnostics, skips disabled hooks without invoking the executor, executes enabled hooks through `HookExecutorPort`, and records request/completion events in the session ledger. +- FAIL hook diagnostics now block both planning and execution at the application layer. +- Shell-backed hook command execution and approval/risk policy wiring remain deferred; unit tests use a fake executor only. + +Phase 17 implemented: +- Added `NoopHookExecutor`, a safe infrastructure adapter for `HookExecutorPort` that records successful hook execution results without invoking shell commands. +- `ExecuteChatToolUseCase` now accepts an optional `ExecuteChatHookUseCase` as `hook_runner`. +- When a hook runner is injected, `pre_tool` hook execution events are recorded before tool execution and `post_tool` hook execution events are recorded after tool execution. +- Existing `PlanChatHooksUseCase` injection behavior remains unchanged for planning-only paths. +- Session ledger ordering is preserved across hook execution request/completion events, diff/tool events, verification events, and post-hook execution events. +- Real shell-backed hook command execution remains deferred until explicit approval/risk policy is wired. + +Phase 18 implemented: +- Added `ShellHookExecutor`, which implements `HookExecutorPort` by delegating hook commands to LAEE `LocalExecutorPort` as `shell_exec` actions. +- Hook shell execution uses the workspace root as `cwd` and includes a configurable timeout. +- LAEE `SUCCESS` observations are normalized to successful `HookExecutionResult` values; `DENIED` / `ERROR` observations become failed hook results with stderr summaries. +- `ExecuteChatToolUseCase` now stops before the tool body when an injected `pre_tool` hook runner records a failed hook result. +- Post-tool hook failure is intentionally recorded as ledger data only; no rollback behavior was added. + +Phase 19 implemented: +- Added a Chat CLI hook executor factory with safe no-op default. +- `MORPHIC_CHAT_HOOK_EXECUTION=shell` explicitly opts into `ShellHookExecutor`; unset, empty, or `noop` uses `NoopHookExecutor`. +- Shell hook executor construction reuses LAEE approval mode, audit log path, and undo settings from the app container when available, with conservative local defaults otherwise. +- Unknown hook execution modes raise a validation error. +- `morphic chat --doctor --json` now reports `hook_execution_mode`. + +Phase 20 implemented: +- Added `morphic hooks run ` for explicit manual hook execution. +- Manual hook runs create a chat session ledger under `.morphic/sessions/*.jsonl` and record hook execution events. +- `morphic hooks run --json` emits session id, hook execution mode, diagnostics, events, results, and summary counts. +- Default manual hook execution remains no-op unless the user explicitly sets `MORPHIC_CHAT_HOOK_EXECUTION=shell`. +- Shell opt-in manual validation is covered by a test that runs `echo hook-ok` through LAEE `shell_exec` and verifies `.morphic/audit_log.jsonl`. + +Phase 21 implemented: +- Added `/hooks run ` handling inside `morphic chat`. +- REPL hook runs record both the slash command and hook execution events in the current chat session ledger. +- REPL hook execution preserves the no-op default and respects `MORPHIC_CHAT_HOOK_EXECUTION=shell`. +- Shell opt-in REPL validation is covered by a test that runs `echo repl-hook-ok` through LAEE and verifies the audit log exists. +- Deferred `D008 Hook command execution` is now complete; broader chat tool execution UX is split into a separate follow-up. + +Phase 22 implemented: +- Added `NoopToolExecutor`, a safe default `ToolExecutorPort` adapter that records successful tool results without invoking local tools. +- Added `/tools run [json_arguments]` handling inside `morphic chat`. +- REPL tool runs record the slash command and route through `ExecuteChatToolUseCase`, producing `tool_call_requested` / `tool_call_completed` events in the current session ledger. +- Existing hook runner flow is injected around REPL tool runs, so configured `pre_tool` / `post_tool` hooks are recorded. +- Invalid JSON tool arguments return a user-facing message without crashing the REPL. +- This starts `D011 General chat tool execution UX beyond explicit hook commands`; LAEE-backed real tool execution remains a follow-up. + +Phase 23 implemented: +- Added a Chat CLI tool executor factory with safe no-op default. +- `MORPHIC_CHAT_TOOL_EXECUTION=laee` explicitly opts into `LaeeToolExecutor`; unset, empty, or `noop` uses `NoopToolExecutor`. +- LAEE tool executor construction reuses the same local executor settings as shell-backed hooks. +- Unknown tool execution modes raise a validation error and surface through `morphic chat --doctor --json` with exit code 2. +- `morphic chat --doctor --json` now reports `tool_execution_mode`. +- REPL `/tools run shell_exec {"cmd":"echo tool-ok"}` is covered by an opt-in test that verifies `.morphic/audit_log.jsonl`. + +Phase 24 implemented: +- `/tools run` now reports LAEE denied/error results with `success=False`, `exit_code`, and a visible `error=` summary from stderr. +- REPL tool execution now assesses risk with `RiskAssessor` from the tool name and JSON arguments before calling `ExecuteChatToolUseCase`. +- `MORPHIC_CHAT_TOOL_EXECUTION=laee` `/tools run fs_delete ...` is covered by a deny-path test that verifies the target file is preserved and `.morphic/audit_log.jsonl` records the denied action. + +Phase 25 implemented: +- Added `--permission-mode` to both `morphic chat` and `morphic code`. +- Selected modes use the existing `PermissionMode` values and are persisted in session start ledger events. +- `/status` already surfaces the active permission mode, so `morphic chat --permission-mode read-only` now reports `mode=read-only`. +- Read-only mutating `/tools run` attempts now return a user-facing `permission denied: ...` assistant message instead of crashing the REPL. + +Phase 26 implemented: +- Added `RouteChatDirectRuntime`, which delegates a Chat CLI turn to exactly one `RouteToEngineUseCase` execution and records the result as one `IMPLEMENTER` council turn plus the assistant decision. +- Added `--route-direct` to both `morphic chat` and `morphic code`; optional `--engine ` pins the preferred route engine while omission keeps automatic routing. +- `--route-direct` and `--route-council` are mutually exclusive, and invalid direct engine ids return exit code 2 diagnostics. +- Direct route failure and empty output are surfaced as errors instead of silently falling back to deterministic local success text. +- Direct external-engine execution currently requires explicit `--permission-mode danger-full-access`; this temporary gate remains until Morphic permission modes propagate to native CLI permission controls. +- Strategy memory added at `.serena/memories/morphic_control_plane_strategy.md`: Morphic competes as a native-engine-preserving multi-engine control plane, not as another single-engine CLI clone. + +Phase 27 implemented: +- Added strict provider-independent `AgentEngineEvent` / `AgentEngineEventType` domain entities for native agent lifecycle data. +- Added a resilient Codex JSONL parser for thread, turn, command/MCP/web tool, file change, plan, reasoning, assistant, completion, failure, and error records while retaining raw payloads. +- Codex results now expose normalized events, thread/session id, final assistant message, usage, and malformed-line count through metadata. +- Replaced deprecated Codex `--full-auto` invocation with explicit `--sandbox`; `read-only`, `workspace-write`, and `danger-full-access` map directly from Morphic permission modes. +- Added Codex `--cd ` propagation so `--workspace` cannot silently execute against the process working directory. +- `confirm-destructive` is rejected for direct Codex execution because `codex exec` cannot open an interactive approval prompt. +- Direct route currently requires explicit `--engine codex_cli`; automatic/native routing will reopen after other adapters implement equivalent workspace and permission mappings. + +Phase 28 implemented: +- Direct native runtime turns now retain normalized provider-independent engine events. +- `SendChatMessageUseCase` persists each engine event independently and in order before its council argument, decision, and assistant response. +- Raw provider payloads remain present in each event for audit, replay, and forward compatibility. +- Added `ScopedAgentEnginePort` so workspace and permission propagation is an explicit adapter capability rather than optional arguments that existing drivers could ignore. +- Scoped routing skips unsupported adapters and can fall back to Codex without executing a permission-unaware engine. + +Phase 29 implemented: +- Added an async subprocess streaming path that drains stdout and stderr concurrently and publishes decoded stdout lines before process completion. +- Added stateful Codex JSONL decoding so live events retain provider order and the active thread id. +- Added narrow `StreamingScopedAgentEnginePort` and `StreamingCouncilRuntimePort` capabilities rather than widening all engine/council implementations. +- Direct streaming chat persists the user message before execution and appends each native event immediately through an application-owned ledger sink. +- Final Codex result metadata still contains the complete event list for buffered consumers; streaming chat deliberately avoids persisting that list twice. +- Unit coverage includes a real local subprocess for line/stderr collection; no real Codex task or paid model was invoked. + +Phase 30 implemented: +- `SendChatMessageUseCase` can fan streamed native events out to an optional presentation observer after each ledger append succeeds. +- Observer failures are logged and isolated from the durable audit path and native execution. +- Added `NativeEventProgressRenderer` for concise run/tool/file/plan/completion/error lines in Chat REPL and one-shot code execution. +- Raw provider payloads, generic progress/reasoning, unknown events, and assistant messages are not rendered; event detail is whitespace-normalized and capped at 160 characters. +- The final assistant response remains rendered once through the existing response path. + +Phase 31 implemented: +- `ChatSession` tracks each native engine session id together with the workspace root and permission mode that created it. +- Resume reconstructs native session provenance by replaying persisted `context_indexed` and `engine_event` records. +- Added `ResumableStreamingScopedAgentEnginePort`; routing never treats ordinary streaming support as implicit resume support. +- Codex resume uses the explicit stored thread id and preserves `--sandbox` plus `--cd` scope; it never uses the ambiguous global `--last` session. +- Direct runtime fails before engine execution when stored workspace or permission provenance differs from the current session. + +Phase 32 implemented: +- Buffered and streaming native CLI subprocess paths now handle task cancellation explicitly. +- Cancellation sends `terminate`, waits up to two seconds, and escalates to `kill` only if the process does not exit. +- The original `CancelledError` is re-raised after cleanup, preserving Ctrl-C/caller cancellation semantics. +- Unit tests pin graceful termination and verify that cancellation is not converted into a normal engine result. + +Phase 33 implemented: +- `ClaudeCodeDriver` now implements scoped workspace and permission execution. +- Morphic read-only maps to Claude `plan`, workspace-write to `acceptEdits`, and danger-full-access to explicit `bypassPermissions` plus the dangerous bypass flag. +- `confirm-destructive` is rejected because Claude headless mode cannot preserve an interactive approval prompt. +- Removed the user-only setting source and hard-coded tool allowlist, allowing Claude Code to preserve native project/local settings, CLAUDE.md, skills, hooks, MCP servers, plugins, and tool policy. +- Claude streaming JSONL normalization and native resume remain the next slice; direct Chat CLI mode remains Codex-only until those contracts are implemented. + +Phase 34 implemented: +- Added Claude stream-json normalization for init, assistant text, tool use, tool result, success, and failure records while retaining raw payloads. +- `ClaudeCodeDriver` now implements the same streaming scoped and explicit resume capabilities as Codex. +- Claude session id, output, model, usage, total cost, and malformed-line diagnostics are retained in normalized results. +- Chat CLI direct mode now accepts explicit `--engine claude_code` as well as `codex_cli`; both use the same ledger, progress, provenance, and fail-closed resume path. + +Phase 35 implemented: +- Native resume requests now carry both the provider session id and its owner engine. +- Preferred-engine/resume-engine mismatches are rejected before route construction. +- Every non-owner engine in the fallback chain is skipped before availability checks or execution and recorded as `resume_engine_mismatch`. +- Provider-native session ids can no longer cross from Claude to Codex or Codex to Claude during fallback. + +Phase 36 implemented: +- Streaming turn cancellation now appends `turn_cancelled` after the user message and any native events already delivered to the ledger. +- The original `CancelledError` still propagates after the durable append, allowing the native subprocess cleanup path to retain its cancellation semantics. +- `morphic chat` and `morphic code` print `Cancelled.` and exit 130 on Ctrl-C instead of silently terminating. +- A cancelled turn never emits council decision or assistant success events. +- Verification: 3,523 unit tests passed; repository-wide Ruff clean. + +Phase 37 implemented: +- Added an injectable `ActiveTurnController` that owns at most one interactive turn child task. +- Ctrl-C during an active chat turn cancels that turn, completes provider cleanup and durable `turn_cancelled` recording, then returns to the same REPL. +- The REPL replays the ledger after cancellation so subsequent events continue after every partially persisted event without duplicate sequence numbers. +- Outer asyncio cancellation remains process-level cancellation; idle chat and one-shot code Ctrl-C behavior remain unchanged. +- The previous SIGINT handler is restored after every turn, and repeated cancellation requests do not interrupt cleanup. +- Non-streaming runtimes now persist the user message before execution and append `turn_cancelled` on interruption, matching the streaming replay contract. +- Verification: 3,528 unit tests passed; repository-wide Ruff clean. + +Phase 38 implemented: +- `morphic chat --control` opts an interactive session into a short-lived active-turn control server bound only to `127.0.0.1` on a random port. +- A protocol-versioned, session-scoped descriptor carries the port and random authentication token under `.morphic/control/`; directory/file modes are 0700/0600. +- `morphic chat-control status` and `morphic chat-control cancel` can inspect or cancel the active turn from another terminal, with optional single-session discovery. +- Invalid tokens, mismatched sessions, unsupported commands, and non-loopback descriptors fail closed without cancelling the turn. +- The owned descriptor is removed after normal completion or after cancellation cleanup; the listener is disabled by default and never remains permanently open. +- Verification: 3,536 unit tests passed; repository-wide Ruff clean. + +Phase 39 implemented: +- Added authenticated `morphic chat-control steer ` over the Phase 38 loopback transport. +- Replacement prompts must be non-empty and at most 2048 UTF-8 bytes; invalid input is rejected before cancellation. +- Steer queueing is first-writer-wins, so a later request cannot replace the accepted prompt during cleanup. +- After `turn_cancelled`, the REPL replays the ledger, appends `turn_steered`, and submits the replacement as a normal `user_message`. +- Replay restores the same provider-native session id, workspace, and permission mode before the replacement route executes. +- Slash-prefixed remote prompts bypass local slash command parsing and remain provider input. +- Verification: 3,542 unit tests passed; repository-wide Ruff clean. + +Phase 40 implemented: +- Added an offline evaluator for one immutable task manifest shared by Codex CLI, Claude Code, and Morphic-controlled trials. +- The manifest pins task id, goal, workspace revision, verification checks, handoff assertions, and repetition count; results must contain each arm/trial exactly once. +- Duplicate, missing, mismatched, out-of-range, undeclared-check, undeclared-handoff, and inconsistent recovery records fail closed. +- Reports expose completion, accepted-patch, verification, median elapsed time, mean cost, mean human interventions, recovery, and context-handoff metrics per arm. +- Verification and handoff fidelity are derived from predeclared assertions rather than trusted as arbitrary scores. +- Metric-specific leaders replace a subjective weighted composite score; deterministic JSON has sorted keys and no timestamp. +- `morphic benchmark agent-cli --manifest ... --results ... [--json]` reads recorded files only and never launches a native engine or paid API. +- The top-level `benchmarks` package is now included in built distributions. +- Verification: 3,553 unit tests passed; repository-wide Ruff clean; wheel contents verified. + +Phase 41 implemented: +- Added `morphic benchmark agent-cli-record`; its default path returns a deterministic plan and creates no worktree or process. +- Recorder config must cover exactly the three arms plus every declared verification check and handoff assertion. +- Live execution requires `--execute`, `--acknowledge-paid`, and an explicit cost cap covering the configured all-trial maximum estimate. +- Every arm/trial runs at the pinned revision in a unique detached worktree outside the source repository. +- Agent, check, and handoff commands receive argv directly without a shell and use a bounded timeout. +- Worktrees are released in `finally` after normal completion, non-zero command results, or raised runner errors. +- Evidence stores command/output hashes, byte counts, exit/timeout/elapsed data, and passed assertion names; raw prompts and output are not persisted. +- Existing evidence is never overwritten; a mode-0600 temporary file is published through an exclusive atomic hard link and then removed. +- Authorized estimate cap is recorded, while actual provider cost and accepted-patch review remain explicitly pending adjudication. +- Verification: 3,566 unit tests passed; repository-wide Ruff clean; detached-worktree lifecycle, exclusive evidence publication, and wheel contents verified. + +Phase 42 implemented: +- Recorder agent output is parsed in memory into a normalized receipt before raw stdout is discarded. +- Codex receipts require usage plus a model/model hint and recompute cost through the existing deterministic calculator. +- Claude receipts retain the provider-reported total cost; Morphic-controlled commands use a strict `morphic_benchmark_receipt` envelope. +- Provider-specific cost sources, non-negative usage, recalculated Codex cost, and zero parse errors are required. +- Evidence becomes `normalized_receipts` only when every trial has a valid receipt; missing receipts are never converted to zero cost. +- Independent review decisions bind accepted patch, interventions, and recovery to the exact agent argv SHA-256 plus a review artifact SHA-256. +- `morphic benchmark agent-cli-finalize` joins complete evidence/review matrices into Phase 40 observations without starting an agent or paid API. +- Finalization recomputes check/handoff outcomes and rejects identity/provider/fingerprint mismatch, missing/duplicate trials, failed-run acceptance, and authorized-cap overage. +- Final output is timestamp-free, sorted-key JSON and uses the existing exclusive non-overwriting publication path. +- Verification: 3,582 unit tests passed; repository-wide Ruff clean; all four benchmark modules are present in the built wheel. + +Phase 43 implemented: +- Added explicit `morphic code --benchmark-receipt` while preserving ordinary output when the flag is absent. +- The final stdout line is a sorted canonical Morphic receipt that aggregates council-turn cost and non-negative normalized completion usage only. +- Runtime failure and Ctrl-C emit no receipt, so unknown provider cost cannot be misreported as zero and finalization fails closed. +- Added parseable manifest and recorder configuration examples under `benchmarks/templates/`. +- Added `morphic benchmark agent-cli-rehearse`, whose commands are fixed internal Python fixtures and cannot be replaced with external agent commands. +- The rehearsal pins a Git revision and exercises three detached worktrees, all receipt parsers, hashed evidence, review fingerprints, and deterministic finalization at configured and actual cost $0. +- Rehearsal review fixes `accepted_patch=false`; fixture completion is never represented as patch-quality evidence. +- Output is a new five-file manifest/config/evidence/reviews/results directory; an existing directory is refused. +- Real workspace verification completed all three cells with no raw output in evidence and no remaining detached worktree. No paid campaign was executed. +- Verification: 3,591 unit tests passed; repository-wide Ruff clean; wheel contains the rehearsal module and both JSON templates. + +Phase 44 implemented: +- Added `morphic benchmark agent-cli-preflight` for a deterministic, non-authorizing campaign artifact. +- Manifest revision must equal the full Git-resolved 40-character commit; symbolic or mismatched revisions fail closed. +- Operator-declared runtime versions must cover exactly three arms and match each configured command executable. +- Runtime versions and every arm/check/handoff command receive deterministic SHA-256 fingerprints. +- Complete manifest/config hashes bind goal, timeout, and model hints without exposing the raw goal. +- Preflight runs only read-only Git revision resolution; no version command, agent, or paid API is started. +- Added `agent-cli-review-template` with every expected arm/trial and null human judgment fields. +- Templates bind exact preflight, normalized evidence, and expanded agent argv SHA-256 values and remain `review_completed=false` until filled. +- Completed bound reviews use `review_completed=true`; finalization checks evidence binding and optional `--preflight` binding. +- Binding fields remain optional for backward compatibility with Phase 42 review artifacts. +- Real zero-cost verification generated a three-arm preflight and three-decision review template on commit `88326ae`; no paid campaign was executed. +- Verification: 3,605 unit tests passed; repository-wide Ruff clean; wheel contains the preflight module and runtime-version template. + +Phase 45 implemented: +- Added a normalized reviewer policy declaration with operator ID, reviewer allowlist, and minimum distinct reviewer count. +- Policy SHA-256 is bound into pending templates and completed reviews. +- Operator self-review, unauthorized reviewers, insufficient diversity, and impossible reviewer minimums fail closed. +- Reviewer IDs are structural declarations only; no authenticated human identity claim is made. +- Added read-only `morphic benchmark agent-cli-status` with six deterministic lifecycle stages. +- Status validates artifact order, manifest/preflight hashes, evidence estimate/matrix, all review bindings, reviewer policy, and recomputed final results. +- Every status fixes `paid_execution_authorized=false` and cannot start a process or mutate an artifact. +- Real zero-cost verification reached `review_pending` at commit `d88f1f0`; all input artifact SHA-256 values were identical before and after status. +- No external agent, version probe, or paid API was started. +- Verification: 3,621 unit tests passed; repository-wide Ruff clean; wheel contains the campaign status, reviewer policy, and policy template artifacts. + +Phase 46 implemented: +- Added self-fingerprinted reviewer trust declarations bound to one benchmark and exact review policy. +- Trust roots contain reviewer ID, globally unique key ID, Ed25519 public key, public-key SHA-256, and active/revoked status. +- Every allowed reviewer needs an active key for new trust authoring; revoked historical keys remain representable for rotation and verification refusal. +- Trust-bound completed reviews retain the exact reviewer trust SHA-256 in addition to preflight, evidence, and review-policy bindings. +- Added `morphic benchmark agent-cli-attestation-template`, which emits one canonical signing payload per distinct reviewer without reading a private key. +- Each statement binds benchmark/task/revision, preflight, evidence, policy, trust, the complete reviews artifact, and that reviewer's decision subset. +- Finalization requires one valid detached Ed25519 signature per distinct reviewer when a review is trust-bound. +- Unknown keys, revoked keys, invalid signatures, missing reviewers, and mixed review/policy/trust artifacts fail closed. +- Campaign status adds `review_attestation_pending`; only a fully verified bundle advances to `review_complete` and `finalized`. +- Legacy unsigned reviews and six-stage unsigned campaigns retain their previous behavior. +- Verification: 3,633 unit tests passed; repository-wide Ruff clean; wheel contains the attestation module and reviewer trust template and declares `cryptography>=46.0.5` directly. +- No external agent, private-key file, version probe, or paid API was started. +- Security boundary: signatures prove possession of a trust-enrolled key, not real-world identity or independently anchored key enrollment. + +Phase 47 implemented: +- Added a normalized, self-fingerprinted offline Ed25519 organization authority declaration. +- Anchored reviewer trust includes the exact authority SHA-256; unanchored Phase 46 trust fingerprints remain backward compatible. +- Added canonical enrollment signing requests for every reviewer key without reading an authority private key. +- Authority enrollment certificates bind authority, benchmark, policy, exact trust artifact, reviewer/key identity, and reviewer public-key fingerprint. +- Every active or revoked key retained in trust must have exactly one valid authority certificate; missing, duplicate, mixed, or invalid certificates fail closed. +- Authority-bound finalization requires both the authority declaration and complete enrollment bundle in addition to Phase 46 reviewer attestations. +- Added a campaign envelope signing request covering manifest, preflight, evidence, reviews, policy, trust, enrollments, attestations, results, and immutable campaign identity. +- The envelope fixes `paid_execution_authorized=false`; successful provenance verification never authorizes a recording run. +- Authority-bound status adds `reviewer_enrollment_pending` and `campaign_envelope_pending` and reaches `finalized` only after the external authority signature verifies. +- Added `agent-cli-reviewer-enrollment-template` and `agent-cli-campaign-envelope-template`; both exclusively publish deterministic signing payloads and never read private keys. +- Legacy unsigned and unanchored signed campaigns retain their previous lifecycle and finalization behavior. +- Verification: 3,646 unit tests passed; repository-wide Ruff clean; wheel contains the authority module plus authority and anchored-trust templates. +- Real Ed25519 enrollment, reviewer attestation, and final envelope signatures were exercised with in-memory keys only. No external authority, agent, network identity provider, or paid API was started. +- Security boundary: authority root distribution, certificate expiry, authority revocation/rotation, and transparency logging are not yet anchored. + +Phase 48 implemented: +- Added a versioned authority-root ledger: genesis is out-of-band and each later root carries an exact rotation statement signed by its immediate predecessor. +- The active Ed25519 root signs the self-fingerprinted generation/revocation ledger; gaps, reuse, unknown revocations, invalid rotations, tampering, and revoked active roots fail closed. +- Reviewer trust, finalization, and campaign envelopes can bind the exact root-ledger SHA-256 while Phase 47 fingerprints and signing bytes remain backward compatible when the field is absent. +- Added RFC 6962-style domain-separated SHA-256 Merkle roots, active-root-signed tree heads, and inclusion audit paths for campaign artifacts. +- Complete old/new transparency logs validate append-only growth by requiring the old entry sequence to be an exact prefix of the new log. +- Ledger-bound status adds `authority_root_pending` and `transparency_pending`; the exact signed envelope must have a valid inclusion proof before `finalized`. +- Added private-key-free rotation, ledger, and tree-head signing templates plus offline log and inclusion-proof CLI commands. +- Verification: 3,655 unit tests passed; repository-wide Ruff clean; no external authority, log service, identity provider, agent, version probe, or paid API ran. +- Security boundary: genesis delivery and compromise reset remain out-of-band; complete-log prefix verification is not a compact consistency proof or gossip protocol. + +Phase 49 implemented: +- Added RFC 6962 minimal consistency proofs using the specified `SUBPROOF` recursion and compact SHA-256 node paths. +- Verification reconstructs both prior and current signed roots without requiring either complete log and rejects size, log, ledger, path, fingerprint, root, or signature mismatch. +- Added self-fingerprinted Ed25519 witness trust with globally unique keys, active/revoked status, and a strict-majority quorum so any two accepted quorums intersect. +- Witness signatures bind the exact log, old/new sizes and roots, both signed tree-head fingerprints, authority-root ledger, consistency proof, and witness trust. +- Missing quorum, duplicate witnesses, unknown/revoked keys, invalid signatures, and same-size different-root split views fail closed. +- Opt-in witnessed campaign status adds `witness_pending`; Phase 48 inclusion-only campaigns remain backward compatible and still finalize without witness inputs. +- Added offline consistency-proof, witness-trust, and checkpoint-template CLI paths plus a parseable three-witness example declaration. +- Verification: 3,664 unit tests passed; repository-wide Ruff clean; RFC seven-leaf proof shape, all prefix sizes through twelve leaves, and the committed witness template were exercised. +- Security boundary: witnesses exchange artifacts out-of-band; Morphic does not yet run a gossip network, attest real-world witness identities, or persist a global checkpoint registry. + +Phase 50 implemented: +- Added a deterministic append-only checkpoint registry whose records bind sequence, prior-record hash, root ledger, witness trust, compact proof, and signed checkpoint. +- Every replay verifies the complete record hash chain plus authority, consistency-proof, and witness signatures; sequence gaps, altered records/links, partial tails, stale extensions, and witnessed split views fail closed. +- Registry append uses an exclusive file lock, `O_APPEND`, `fsync`, regular-file checks, and mode 0600; concurrent duplicate appends and authenticated retry are idempotent. +- Added self-fingerprinted Ed25519 peer trust with globally unique key IDs, active/revoked rotation, and at least one active key per peer. +- Signed exchange packets bind the source peer to one exact registry record/checkpoint/root; import authenticates the peer before using the same locked conflict-safe append path. +- Added private-key-free peer-trust, registry status/store/export-template/import CLI paths and a parseable peer-rotation example. +- Verification: 3,679 unit tests passed; repository-wide Ruff clean; in-memory keys and temporary local files only. +- Security boundary: no online listener, peer discovery, real-world identity attestation, global consensus, or atomic multi-record range sync is provided. + +Phase 51 implemented: +- Added bounded signed ranges that bind base/head hashes, contiguous sequences, every record fingerprint, registry, source peer, and peer trust. +- Range import authenticates and validates the complete range before locking, exact-matches existing overlap, and appends only a missing contiguous suffix. +- Gap, fork, conflicting overlap, invalid signature, or invalid checkpoint chain causes zero mutation; process-level write failure truncates to the original registry size. +- Added receiver-signed acknowledgements bound to the exact range bundle and applied record/tree head. +- Added a mode-0600, locked, hash-chained peer cursor ledger with exact retry idempotency and monotonic positions per source/receiver pair. +- Cursor regression, same-sequence conflicts, invalid acknowledgement signatures, fingerprint tampering, and chain tampering fail closed. +- Added private-key-free range export/import, acknowledgement-template, cursor store, and cursor status CLI paths. +- Verification: 3,686 unit tests passed; repository-wide Ruff clean; no external agent, peer listener, or paid API ran. +- Security boundary: peer trust remains a single snapshot; old trust artifacts are required to replay historical acknowledgements after rotation. + +Phase 52 implemented: +- Added generation-two-and-later peer-trust rollover statements approved by a strict majority of distinct active predecessor peers. +- Rotation statements bind registry, generation, predecessor/successor trust fingerprints, and the automatically computed quorum. +- Private-key-free templates emit one signing request per predecessor peer and allow any active key registered for that peer. +- Minority, duplicate peer, successor-only/revoked key, invalid signature, generation gap/reorder, registry change, trust reuse, and tampering fail closed. +- Added a contiguous self-fingerprinted peer-trust generation ledger rooted in out-of-band genesis. +- Historical trust resolution lets one cursor chain verify acknowledgements from before and after key rotation. +- Cursor store/status accepts exactly one `--peer-trust` or `--peer-trust-ledger`; rotation-template and ledger CLI paths were added. +- Verification: 3,691 unit tests passed; repository-wide Ruff clean; in-memory signatures only, with no external peer, listener, agent, or paid API. +- Security boundary: genesis delivery, latest-ledger pinning, stale-ledger rollback detection, real-world identity, and transport remain out-of-band. + +Phase 53 implemented: +- Added an explicit opt-in checkpoint gossip listener bound only to `127.0.0.1` on a random port. +- A protocol-versioned mode 0600 descriptor binds registry, source peer, random instance, and a 32-byte bearer token. +- Every connection uses one-use 32-byte client/server nonces and HMAC-SHA256 over protocol, endpoint identity, operation, and payload; responses are authenticated too. +- Reused nonces, wrong tokens, endpoint/version mismatch, oversized requests, exhausted bounds, timeout, and invalid artifacts fail closed. +- Fixed limits cover 64 KiB requests, 2 MiB responses, eight concurrent clients, 1,024 retained nonces/requests, two-second request handling, and an operator-bounded listener lifetime. +- The server verifies pre-signed exact range bundles before listening; fetch clients independently verify them against a trust snapshot or generation ledger. +- Submitted signed acknowledgements pass through the existing signature, monotonicity, locking, and hash-chained cursor-store path. +- Added explicit serve/status/fetch/ack CLI paths; max request count, lifetime, cancellation, and context exit close listeners, active writers/tasks, and the owned descriptor. +- Verification: 3,697 unit tests passed; repository-wide Ruff and focused mypy clean; no external peer, agent, paid API, or non-loopback socket ran. +- Security boundary: the descriptor token authenticates local transport only; full witness/root verification occurs during registry import. Remote TLS, discovery, automatic signing, and a durable catch-up loop remain out-of-scope. + +Phase 54 implemented: +- Added a bounded resumable pull loop that selects a signed range containing the exact next local registry sequence and atomically imports only the missing suffix. +- A mode 0600 JSONL sync audit holds one non-blocking exclusive lock for the whole loop and fsyncs a self-fingerprinted hash chain. +- Deterministic audit events bind the exact loop-policy fingerprint and cover imported ranges, safe retry metadata, registry-ahead crash recovery, trust advancement, and explicit stop reasons; no token, timestamp, or raw exception text is persisted. +- The verified checkpoint registry remains import truth. If it is ahead of audit after a crash, the pinned historical registry head must match before a recovered event resumes from the current head. +- Every audit record pins peer-trust generation/trust/ledger fingerprints. Older ledgers and forks at a pinned generation are rejected before contacting the gossip endpoint; contiguous extensions advance the pin. +- Loop policy bounds rounds, newly imported records, request attempts, and deterministic backoff. Stops report up-to-date, range-gap, record-budget, round-budget, or retry-exhausted. +- Added a private-key-free gossip-sync CLI using the existing range signature, root/witness/Merkle, registry, and trust-ledger verification paths. +- Verification: 3,698 unit tests passed; repository-wide Ruff and focused mypy clean; loopback/in-memory test keys only, with no external peer, agent, paid API, or non-loopback listener. +- Security boundary: the local pin is tamper-evident, not resistant to an attacker who can rewrite the complete audit and registry. Automatic acknowledgement signing, peer discovery, and remote mTLS remain out-of-scope. + +Phase 55 implemented: +- Added private-key-free TLS leaf enrollment statements binding normalized DER and SPKI SHA-256 pins, subject/issuer, serial, validity, SANs, and client/server EKUs to an existing active peer Ed25519 identity signature. +- Per-peer certificate rotation is a contiguous generation chain. Missing or mismatched predecessors, invalid/inactive identity signatures, registry/trust mismatch, and duplicate active leaf/SPKI pins across peers fail closed. +- Added a self-fingerprinted TLS trust artifact with deterministic active enrollment resolution. Runtime credentials must match both the active certificate and SPKI pins; an older enrolled certificate cannot start a server or authenticate a client. +- Added protocol version 2 transport using TLS 1.3 only with required mutual CA validation. The client additionally verifies hostname, explicit server IP allowlist, connected peer address, descriptor trust fingerprint, and active server pins. +- The server verifies an explicit client IP allowlist, the handshake leaf against an active peer enrollment, and the request's declared client peer ID. Plaintext connections never reach the application protocol and there is no token or plaintext fallback. +- Preserved 64 KiB request, 2 MiB response, eight-client default, 1,024 nonce/request capacity, timeout, replay, and deterministic listener/client/owned-descriptor cleanup bounds. The mode 0600 descriptor is token-free and TLS private keys must deny group/other access. +- Added private-key-free enrollment-template, detached-signature enrollment finalization, TLS trust, mTLS serve, and mTLS status CLI paths. Loaded trust artifacts are reverified against the exact peer trust before CUI network use. +- Verification: 3,700 unit tests passed; repository-wide Ruff, focused mypy, and wheel build are clean. Tests use loopback, local temporary certificates, and in-memory Ed25519 keys only. +- Security boundary: CA/genesis trust distribution, DNS/address operations, certificate revocation status, private-key custody, peer discovery, automatic signing, and mTLS fetch/ack/sync CLI wiring remain operator/out-of-scope boundaries. + +Phase 56 implemented: +- Added a typed authenticated request-sender protocol beneath status, signed range fetch, and signed acknowledgement submission. Omitting it preserves the existing protocol-v1 loopback behavior and public callers. +- Added a reusable protocol-v2 mTLS client that retains one exact descriptor, client peer, TLS trust, leaf/key/CA, hostname, server address allowlist, and timeout configuration. +- Artifact validation remains transport-independent: fetched ranges are still reverified under their exact peer-trust generation, and returned cursor records must contain the exact submitted acknowledgement. +- The Phase 54 bounded resumable sync loop now accepts the same sender for status and fetch operations. Locking, deterministic audit records, crash recovery, retry delays, budgets, trust rollback/fork rejection, registry import, and stop reasons are unchanged. +- Added mTLS fetch, acknowledgement, and sync CLI paths. Each rebuilds TLS trust against either the supplied peer-trust snapshot or the active generation of the supplied ledger before network access. +- The token-free descriptor, TLS 1.3-only mutual authentication, hostname/address checks, active DER/SPKI pins, nonce replay defense, message limits, and deterministic cleanup remain the protocol-v2 boundary. +- Verification: 3,700 unit tests passed; repository-wide Ruff, focused mypy, and wheel build are clean. +- Security boundary: handshake expiry is enforced by TLS, but signed explicit revocation, pre-expiry warnings/grace periods, OCSP/CRL policy, peer discovery, and automatic enrollment/ack signing remain out of scope. + Key design decisions: - Start with a line-oriented `morphic chat` REPL; defer full-screen Textual UI until the event/session model is stable. - `.morphic/` becomes the canonical workspace metadata layer over time. @@ -111,7 +515,7 @@ Key design decisions: - Existing `specs/council-pilot/` remains the lower-level two-engine debate spike; `morphic-chat-cli` is the higher-level terminal UX and harness. Recommended next implementation step: -- Continue Deferred `D008` by adding an approved hook command executor or wiring hook planner construction into future CLI tool execution paths. +- Add peer-signed certificate revocation and deterministic expiry policy before authenticated discovery, without weakening active DER/SPKI pins or allowing transport downgrade. > Last updated: 2026-05-20 > Last commit: `feat(router): Goal Classifier Router for planner model selection (TD-195)` diff --git a/domain/entities/agent_engine_event.py b/domain/entities/agent_engine_event.py new file mode 100644 index 0000000..c9cc82f --- /dev/null +++ b/domain/entities/agent_engine_event.py @@ -0,0 +1,42 @@ +"""Normalized event vocabulary for native agent engine runs.""" + +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from domain.value_objects.agent_engine import AgentEngineType + + +class AgentEngineEventType(str, Enum): + """Provider-independent lifecycle events emitted by native agent engines.""" + + RUN_STARTED = "run_started" + TURN_STARTED = "turn_started" + PROGRESS = "progress" + TOOL_STARTED = "tool_started" + TOOL_COMPLETED = "tool_completed" + FILE_CHANGED = "file_changed" + PLAN_UPDATED = "plan_updated" + ASSISTANT_MESSAGE = "assistant_message" + RUN_COMPLETED = "run_completed" + RUN_FAILED = "run_failed" + ERROR = "error" + UNKNOWN = "unknown" + + +class AgentEngineEvent(BaseModel): + """One normalized native-engine event with the raw provider payload retained.""" + + model_config = ConfigDict(strict=True, validate_assignment=True, frozen=True) + + type: AgentEngineEventType + engine: AgentEngineType + sequence: int = Field(ge=0) + session_id: str | None = None + item_id: str | None = None + item_type: str | None = None + text: str | None = None + payload: dict[str, Any] = Field(default_factory=dict) diff --git a/domain/entities/chat_event.py b/domain/entities/chat_event.py index d3f5f76..7cd5dc9 100644 --- a/domain/entities/chat_event.py +++ b/domain/entities/chat_event.py @@ -21,6 +21,9 @@ class ChatEventType(str, Enum): COUNCIL_STARTED = "council_started" COUNCIL_ARGUMENT = "council_argument" COUNCIL_DECISION = "council_decision" + ENGINE_EVENT = "engine_event" + TURN_CANCELLED = "turn_cancelled" + TURN_STEERED = "turn_steered" TOOL_CALL_REQUESTED = "tool_call_requested" APPROVAL_REQUESTED = "approval_requested" APPROVAL_RESOLVED = "approval_resolved" @@ -30,6 +33,8 @@ class ChatEventType(str, Enum): VERIFICATION_RESULT = "verification_result" HOOK_EXECUTION_PLANNED = "hook_execution_planned" HOOK_EXECUTION_SKIPPED = "hook_execution_skipped" + HOOK_EXECUTION_REQUESTED = "hook_execution_requested" + HOOK_EXECUTION_COMPLETED = "hook_execution_completed" MEMORY_CANDIDATE = "memory_candidate" SESSION_SUMMARY = "session_summary" SESSION_ENDED = "session_ended" diff --git a/domain/entities/chat_session.py b/domain/entities/chat_session.py index 2610470..8248d9a 100644 --- a/domain/entities/chat_session.py +++ b/domain/entities/chat_session.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field from domain.entities.chat_event import ChatEvent, ChatEventType +from domain.value_objects.agent_engine import AgentEngineType class PermissionMode(str, Enum): @@ -30,6 +31,17 @@ class ChatSessionStatus(str, Enum): ENDED = "ended" +class NativeEngineSession(BaseModel): + """Native thread identity bound to its original safety scope.""" + + model_config = ConfigDict(strict=True, validate_assignment=True, frozen=True) + + engine: AgentEngineType + session_id: str = Field(min_length=1) + workspace_root: str = Field(min_length=1) + permission_mode: PermissionMode + + class ChatSession(BaseModel): """Session state used to sequence append-only chat events.""" @@ -39,6 +51,8 @@ class ChatSession(BaseModel): status: ChatSessionStatus = ChatSessionStatus.ACTIVE goal: str | None = Field(default=None, min_length=1) permission_mode: PermissionMode = PermissionMode.CONFIRM_DESTRUCTIVE + workspace_root: str | None = Field(default=None, min_length=1) + native_sessions: dict[str, NativeEngineSession] = Field(default_factory=dict) next_sequence: int = Field(default=0, ge=0) @classmethod @@ -69,8 +83,60 @@ def record_event( created_at=created_at or datetime.now(tz=UTC), payload=payload, ) - updated = self.model_copy(update={"next_sequence": self.next_sequence + 1}) + updated = self._apply_event_state( + event_type=event_type, + payload=payload, + next_sequence=self.next_sequence + 1, + ) return updated, event + def replay_event(self, event: ChatEvent) -> ChatSession: + """Apply one already-persisted event without creating a replacement.""" + if event.session_id != self.id: + raise ValueError("cannot replay event from another chat session") + return self._apply_event_state( + event_type=event.type, + payload=event.payload, + next_sequence=max(self.next_sequence, event.sequence + 1), + ) + + def _apply_event_state( + self, + *, + event_type: ChatEventType, + payload: dict[str, Any], + next_sequence: int, + ) -> ChatSession: + updates: dict[str, Any] = {"next_sequence": next_sequence} + if event_type is ChatEventType.CONTEXT_INDEXED: + workspace_root = payload.get("workspace_root") + if isinstance(workspace_root, str) and workspace_root: + updates["workspace_root"] = workspace_root + elif event_type is ChatEventType.ENGINE_EVENT: + engine_value = payload.get("engine") + session_id = payload.get("session_id") + if ( + isinstance(engine_value, str) + and isinstance(session_id, str) + and session_id + and self.workspace_root + ): + try: + engine = AgentEngineType(engine_value) + except ValueError: + pass + else: + native_sessions = dict(self.native_sessions) + native_sessions[engine.value] = NativeEngineSession( + engine=engine, + session_id=session_id, + workspace_root=self.workspace_root, + permission_mode=self.permission_mode, + ) + updates["native_sessions"] = native_sessions + elif event_type is ChatEventType.SESSION_ENDED: + updates["status"] = ChatSessionStatus.ENDED + return self.model_copy(update=updates) + def close(self) -> ChatSession: return self.model_copy(update={"status": ChatSessionStatus.ENDED}) diff --git a/domain/entities/council_runtime.py b/domain/entities/council_runtime.py index bb4cd1a..d00c116 100644 --- a/domain/entities/council_runtime.py +++ b/domain/entities/council_runtime.py @@ -6,6 +6,8 @@ from pydantic import BaseModel, ConfigDict, Field +from domain.entities.agent_engine_event import AgentEngineEvent + class CouncilRole(str, Enum): """Deliberation responsibilities independent from execution engines.""" @@ -28,6 +30,7 @@ class CouncilTurn(BaseModel): engine_id: str = Field(min_length=1) content: str = Field(min_length=1) evidence: list[str] = Field(default_factory=list) + engine_events: list[AgentEngineEvent] = Field(default_factory=list) cost_usd: float = Field(default=0.0, ge=0.0) latency_ms: int = Field(default=0, ge=0) diff --git a/domain/entities/hook.py b/domain/entities/hook.py index 6f9bf3d..25270a1 100644 --- a/domain/entities/hook.py +++ b/domain/entities/hook.py @@ -2,6 +2,8 @@ from __future__ import annotations +import uuid +from datetime import UTC, datetime from enum import Enum from pydantic import BaseModel, ConfigDict, Field @@ -42,3 +44,30 @@ class HookDiagnostic(BaseModel): message: str = Field(min_length=1) source_path: str | None = Field(default=None, min_length=1) duration_ms: float = Field(default=0.0, ge=0.0) + + +class HookExecutionRequest(BaseModel): + """Normalized request to execute one validated hook command.""" + + model_config = ConfigDict(strict=True, validate_assignment=True, frozen=True) + + id: str = Field(default_factory=lambda: str(uuid.uuid4()), min_length=1) + session_id: str = Field(min_length=1) + hook_name: str = Field(min_length=1) + hook_type: HookType + command: str = Field(min_length=1) + source_path: str = Field(min_length=1) + requested_at: datetime = Field(default_factory=lambda: datetime.now(tz=UTC)) + + +class HookExecutionResult(BaseModel): + """Normalized result from a hook executor implementation.""" + + model_config = ConfigDict(strict=True, validate_assignment=True, frozen=True) + + request_id: str = Field(min_length=1) + success: bool + stdout_summary: str = "" + stderr_summary: str = "" + exit_code: int | None = None + completed_at: datetime = Field(default_factory=lambda: datetime.now(tz=UTC)) diff --git a/domain/ports/agent_engine.py b/domain/ports/agent_engine.py index a392c57..9ed11e1 100644 --- a/domain/ports/agent_engine.py +++ b/domain/ports/agent_engine.py @@ -11,6 +11,8 @@ from dataclasses import dataclass, field from datetime import UTC, datetime +from domain.entities.agent_engine_event import AgentEngineEvent +from domain.entities.chat_session import PermissionMode from domain.value_objects.agent_engine import AgentEngineType from domain.value_objects.fallback_attempt import FallbackAttempt @@ -68,3 +70,58 @@ async def is_available(self) -> bool: ... @abstractmethod def get_capabilities(self) -> AgentEngineCapabilities: ... + + +class ScopedAgentEnginePort(AgentEnginePort): + """Agent engine that preserves workspace and permission boundaries.""" + + @abstractmethod + async def run_task_scoped( + self, + task: str, + *, + workspace_root: str, + permission_mode: PermissionMode, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: ... + + +class AgentEngineEventSinkPort(ABC): + """Consumes normalized native-engine events as they occur.""" + + @abstractmethod + async def publish(self, event: AgentEngineEvent) -> None: ... + + +class StreamingScopedAgentEnginePort(ScopedAgentEnginePort): + """Scoped engine that can publish events before process completion.""" + + @abstractmethod + async def run_task_scoped_stream( + self, + task: str, + *, + workspace_root: str, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: ... + + +class ResumableStreamingScopedAgentEnginePort(StreamingScopedAgentEnginePort): + """Streaming scoped engine that can continue an explicit native session.""" + + @abstractmethod + async def resume_task_scoped_stream( + self, + task: str, + *, + resume_session_id: str, + workspace_root: str, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: ... diff --git a/domain/ports/council_runtime.py b/domain/ports/council_runtime.py index cc0429f..aefd23e 100644 --- a/domain/ports/council_runtime.py +++ b/domain/ports/council_runtime.py @@ -7,6 +7,7 @@ from domain.entities.chat_session import ChatSession from domain.entities.council_runtime import CouncilDecision, CouncilTurn from domain.entities.workspace_context import ContextIndex +from domain.ports.agent_engine import AgentEngineEventSinkPort class CouncilRuntimePort(ABC): @@ -19,3 +20,16 @@ async def deliberate( context: ContextIndex, user_message: str, ) -> tuple[list[CouncilTurn], CouncilDecision]: ... + + +class StreamingCouncilRuntimePort(CouncilRuntimePort): + """Council-compatible runtime that emits native events during execution.""" + + @abstractmethod + async def deliberate_stream( + self, + session: ChatSession, + context: ContextIndex, + user_message: str, + event_sink: AgentEngineEventSinkPort, + ) -> tuple[list[CouncilTurn], CouncilDecision]: ... diff --git a/domain/ports/hook_executor.py b/domain/ports/hook_executor.py new file mode 100644 index 0000000..05cd3fd --- /dev/null +++ b/domain/ports/hook_executor.py @@ -0,0 +1,14 @@ +"""Port for approved chat hook command execution.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from domain.entities.hook import HookExecutionRequest, HookExecutionResult + + +class HookExecutorPort(ABC): + """Executes one normalized hook command through an infrastructure adapter.""" + + @abstractmethod + async def execute(self, request: HookExecutionRequest) -> HookExecutionResult: ... diff --git a/infrastructure/agent_cli/_subprocess_base.py b/infrastructure/agent_cli/_subprocess_base.py index 0906e5c..137ae81 100644 --- a/infrastructure/agent_cli/_subprocess_base.py +++ b/infrastructure/agent_cli/_subprocess_base.py @@ -8,6 +8,7 @@ import asyncio import shutil +from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -28,6 +29,7 @@ async def _run_cli( cmd: list[str], timeout: float = 300.0, env: dict[str, str] | None = None, + cwd: str | None = None, ) -> CLIResult: """Run a CLI command asynchronously with timeout. @@ -44,6 +46,7 @@ async def _run_cli( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env, + cwd=cwd, ) try: stdout_bytes, stderr_bytes = await asyncio.wait_for( @@ -55,6 +58,9 @@ async def _run_cli( stderr=stderr_bytes.decode(errors="replace"), returncode=proc.returncode or 0, ) + except asyncio.CancelledError: + await self._terminate_process(proc) + raise except TimeoutError: proc.kill() await proc.communicate() @@ -64,6 +70,78 @@ async def _run_cli( returncode=-1, ) + async def _run_cli_streaming( + self, + cmd: list[str], + *, + timeout: float = 300.0, + on_stdout_line: Callable[[str], Awaitable[None]], + env: dict[str, str] | None = None, + cwd: str | None = None, + ) -> CLIResult: + """Run a CLI while delivering decoded stdout lines incrementally.""" + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + cwd=cwd, + ) + assert proc.stdout is not None + assert proc.stderr is not None + + async def read_stdout() -> str: + chunks: list[str] = [] + while line := await proc.stdout.readline(): + decoded = line.decode(errors="replace") + chunks.append(decoded) + await on_stdout_line(decoded.rstrip("\r\n")) + return "".join(chunks) + + async def read_stderr() -> str: + return (await proc.stderr.read()).decode(errors="replace") + + try: + stdout, stderr, _ = await asyncio.wait_for( + asyncio.gather(read_stdout(), read_stderr(), proc.wait()), + timeout=timeout, + ) + return CLIResult( + stdout=stdout, + stderr=stderr, + returncode=proc.returncode or 0, + ) + except asyncio.CancelledError: + await self._terminate_process(proc) + raise + except TimeoutError: + proc.kill() + await proc.communicate() + return CLIResult( + stdout="", + stderr=f"Command timed out after {timeout}s", + returncode=-1, + ) + except Exception: + proc.kill() + await proc.communicate() + raise + + async def _terminate_process( + self, + proc: asyncio.subprocess.Process, + grace_seconds: float = 2.0, + ) -> None: + """Stop a cancelled child without swallowing caller cancellation.""" + if proc.returncode is not None: + return + proc.terminate() + try: + await asyncio.wait_for(proc.wait(), timeout=grace_seconds) + except TimeoutError: + proc.kill() + await proc.wait() + @staticmethod def _check_cli_exists(binary: str) -> bool: """Check if a CLI binary is available on PATH.""" diff --git a/infrastructure/agent_cli/claude_code_driver.py b/infrastructure/agent_cli/claude_code_driver.py index 5b92886..dd9d963 100644 --- a/infrastructure/agent_cli/claude_code_driver.py +++ b/infrastructure/agent_cli/claude_code_driver.py @@ -5,17 +5,20 @@ import json import time +from domain.entities.chat_session import PermissionMode from domain.ports.agent_engine import ( AgentEngineCapabilities, - AgentEnginePort, + AgentEngineEventSinkPort, AgentEngineResult, + ResumableStreamingScopedAgentEnginePort, ) from domain.services.engine_cost_calculator import EngineCostCalculator from domain.value_objects.agent_engine import AgentEngineType from infrastructure.agent_cli._subprocess_base import SubprocessMixin +from infrastructure.agent_cli.claude_jsonl import ClaudeJsonlEventDecoder, parse_claude_output -class ClaudeCodeDriver(SubprocessMixin, AgentEnginePort): +class ClaudeCodeDriver(SubprocessMixin, ResumableStreamingScopedAgentEnginePort): """Agent engine backed by Claude Code CLI (headless). Executes `claude -p --output-format json` and parses structured output. @@ -33,6 +36,82 @@ async def run_task( task: str, model: str | None = None, timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=None, + permission_mode=PermissionMode.READ_ONLY, + ) + + async def run_task_scoped( + self, + task: str, + *, + workspace_root: str, + permission_mode: PermissionMode, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=workspace_root, + permission_mode=permission_mode, + ) + + async def run_task_scoped_stream( + self, + task: str, + *, + workspace_root: str, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=workspace_root, + permission_mode=permission_mode, + event_sink=event_sink, + ) + + async def resume_task_scoped_stream( + self, + task: str, + *, + resume_session_id: str, + workspace_root: str, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=workspace_root, + permission_mode=permission_mode, + event_sink=event_sink, + resume_session_id=resume_session_id, + ) + + async def _run_task( + self, + *, + task: str, + model: str | None, + timeout_seconds: float, + workspace_root: str | None, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort | None = None, + resume_session_id: str | None = None, ) -> AgentEngineResult: if not self._enabled: return AgentEngineResult( @@ -42,26 +121,78 @@ async def run_task( error="Claude Code driver is disabled", ) + claude_mode = self._permission_mode(permission_mode) + if claude_mode is None: + return AgentEngineResult( + engine=AgentEngineType.CLAUDE_CODE, + success=False, + output="", + error=( + "Claude Code headless mode cannot preserve confirm-destructive " + "approval prompts" + ), + ) + cmd = [ self._cli_path, "-p", task, "--output-format", - "json", + "stream-json" if event_sink is not None else "json", "--max-turns", "10", - "--setting-sources", - "user", - "--allowedTools", - "Bash,Read,Write,Edit,WebFetch,WebSearch", + "--permission-mode", + claude_mode, ] + if permission_mode is PermissionMode.DANGER_FULL_ACCESS: + cmd.append("--dangerously-skip-permissions") + if resume_session_id: + cmd.extend(["--resume", resume_session_id]) if model: cmd.extend(["--model", model]) start = time.monotonic() - cli_result = await self._run_cli(cmd, timeout=timeout_seconds) + if event_sink is None: + cli_result = await self._run_cli( + cmd, timeout=timeout_seconds, cwd=workspace_root + ) + else: + decoder = ClaudeJsonlEventDecoder() + + async def publish_line(line: str) -> None: + for event in decoder.decode(line): + await event_sink.publish(event) + + cli_result = await self._run_cli_streaming( + cmd, + timeout=timeout_seconds, + on_stdout_line=publish_line, + cwd=workspace_root, + ) duration = time.monotonic() - start + if event_sink is not None: + parsed = parse_claude_output(cli_result.stdout) + metadata = { + "events": [event.model_dump(mode="json") for event in parsed.events] + } + if parsed.session_id: + metadata["session_id"] = parsed.session_id + if parsed.usage: + metadata["usage"] = parsed.usage + if parsed.parse_errors: + metadata["parse_errors"] = parsed.parse_errors + return AgentEngineResult( + engine=AgentEngineType.CLAUDE_CODE, + success=cli_result.returncode == 0 and parsed.error is None, + output=parsed.output, + error=parsed.error or (cli_result.stderr if cli_result.returncode else None), + cost_usd=parsed.cost_usd, + duration_seconds=duration, + model_used=parsed.model or model, + metadata=metadata, + ) + if cli_result.returncode != 0: return AgentEngineResult( engine=AgentEngineType.CLAUDE_CODE, @@ -99,6 +230,15 @@ async def run_task( metadata=metadata, ) + def _permission_mode(self, permission_mode: PermissionMode) -> str | None: + if permission_mode is PermissionMode.READ_ONLY: + return "plan" + if permission_mode is PermissionMode.WORKSPACE_WRITE: + return "acceptEdits" + if permission_mode is PermissionMode.DANGER_FULL_ACCESS: + return "bypassPermissions" + return None + async def is_available(self) -> bool: return self._enabled and self._check_cli_exists(self._cli_path) diff --git a/infrastructure/agent_cli/claude_jsonl.py b/infrastructure/agent_cli/claude_jsonl.py new file mode 100644 index 0000000..03b0885 --- /dev/null +++ b/infrastructure/agent_cli/claude_jsonl.py @@ -0,0 +1,158 @@ +"""Normalize Claude Code stream-json output into Morphic events.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any + +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType +from domain.value_objects.agent_engine import AgentEngineType + + +@dataclass(frozen=True) +class ClaudeParsedRun: + output: str + events: list[AgentEngineEvent] = field(default_factory=list) + session_id: str | None = None + model: str | None = None + usage: dict[str, int] | None = None + cost_usd: float = 0.0 + error: str | None = None + parse_errors: int = 0 + + +class ClaudeJsonlEventDecoder: + def __init__(self) -> None: + self._sequence = 0 + self.session_id: str | None = None + + def decode(self, line: str) -> list[AgentEngineEvent]: + try: + raw = json.loads(line) + except json.JSONDecodeError: + return [] + if not isinstance(raw, dict) or "type" not in raw: + return [] + session_id = raw.get("session_id") + if isinstance(session_id, str) and session_id: + self.session_id = session_id + return self._events(raw) + + def _events(self, raw: dict[str, Any]) -> list[AgentEngineEvent]: + raw_type = raw.get("type") + if raw_type == "system" and raw.get("subtype") == "init": + return [self._event(AgentEngineEventType.RUN_STARTED, raw)] + if raw_type == "result": + event_type = ( + AgentEngineEventType.RUN_FAILED + if raw.get("is_error") or raw.get("subtype") != "success" + else AgentEngineEventType.RUN_COMPLETED + ) + return [self._event(event_type, raw, text=_string(raw.get("result")))] + message = raw.get("message") + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + return [] + events: list[AgentEngineEvent] = [] + for block in content: + if not isinstance(block, dict): + continue + block_type = block.get("type") + if raw_type == "assistant" and block_type == "tool_use": + events.append( + self._event( + AgentEngineEventType.TOOL_STARTED, + raw, + item_id=_string(block.get("id")), + item_type=_string(block.get("name")), + text=_tool_text(block), + ) + ) + elif raw_type == "user" and block_type == "tool_result": + events.append( + self._event( + AgentEngineEventType.TOOL_COMPLETED, + raw, + item_id=_string(block.get("tool_use_id")), + item_type="tool_result", + text=_string(block.get("content")), + ) + ) + elif raw_type == "assistant" and block_type == "text": + events.append( + self._event( + AgentEngineEventType.ASSISTANT_MESSAGE, + raw, + text=_string(block.get("text")), + ) + ) + return events + + def _event( + self, + event_type: AgentEngineEventType, + raw: dict[str, Any], + *, + item_id: str | None = None, + item_type: str | None = None, + text: str | None = None, + ) -> AgentEngineEvent: + event = AgentEngineEvent( + type=event_type, + engine=AgentEngineType.CLAUDE_CODE, + sequence=self._sequence, + session_id=self.session_id, + item_id=item_id, + item_type=item_type, + text=text, + payload=raw, + ) + self._sequence += 1 + return event + + +def parse_claude_output(stdout: str) -> ClaudeParsedRun: + decoder = ClaudeJsonlEventDecoder() + events: list[AgentEngineEvent] = [] + decoded: list[dict[str, Any]] = [] + parse_errors = 0 + for line in stdout.splitlines(): + try: + raw = json.loads(line) + except json.JSONDecodeError: + parse_errors += 1 + continue + if isinstance(raw, dict): + decoded.append(raw) + events.extend(decoder.decode(line)) + result = next((item for item in reversed(decoded) if item.get("type") == "result"), {}) + init = next((item for item in decoded if item.get("type") == "system"), {}) + usage = result.get("usage") if isinstance(result.get("usage"), dict) else None + error = None + if result and (result.get("is_error") or result.get("subtype") != "success"): + error = str(result.get("result") or result.get("subtype") or "Claude run failed") + return ClaudeParsedRun( + output=str(result.get("result") or stdout), + events=events, + session_id=decoder.session_id, + model=_string(init.get("model")), + usage=usage, + cost_usd=float(result.get("total_cost_usd") or 0.0), + error=error, + parse_errors=parse_errors, + ) + + +def _tool_text(block: dict[str, Any]) -> str | None: + tool_input = block.get("input") + if isinstance(tool_input, dict): + for key in ("command", "file_path", "path"): + value = _string(tool_input.get(key)) + if value: + return value + return _string(block.get("name")) + + +def _string(value: object) -> str | None: + return value if isinstance(value, str) and value else None diff --git a/infrastructure/agent_cli/codex_cli_driver.py b/infrastructure/agent_cli/codex_cli_driver.py index 7d3ef36..23af5fe 100644 --- a/infrastructure/agent_cli/codex_cli_driver.py +++ b/infrastructure/agent_cli/codex_cli_driver.py @@ -2,23 +2,25 @@ from __future__ import annotations -import json import time +from domain.entities.chat_session import PermissionMode from domain.ports.agent_engine import ( AgentEngineCapabilities, - AgentEnginePort, + AgentEngineEventSinkPort, AgentEngineResult, + ResumableStreamingScopedAgentEnginePort, ) from domain.services.engine_cost_calculator import EngineCostCalculator from domain.value_objects.agent_engine import AgentEngineType from infrastructure.agent_cli._subprocess_base import SubprocessMixin +from infrastructure.agent_cli.codex_jsonl import CodexJsonlEventDecoder, parse_codex_output -class CodexCLIDriver(SubprocessMixin, AgentEnginePort): +class CodexCLIDriver(SubprocessMixin, ResumableStreamingScopedAgentEnginePort): """Agent engine backed by OpenAI Codex CLI. - Executes `codex exec --json --full-auto ` and parses structured output. + Executes `codex exec --json` with an explicit sandbox and parses JSONL output. """ engine_type: AgentEngineType = AgentEngineType.CODEX_CLI @@ -32,6 +34,83 @@ async def run_task( task: str, model: str | None = None, timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=None, + permission_mode=PermissionMode.READ_ONLY, + ) + + async def run_task_scoped( + self, + task: str, + *, + workspace_root: str, + permission_mode: PermissionMode, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=workspace_root, + permission_mode=permission_mode, + event_sink=None, + ) + + async def run_task_scoped_stream( + self, + task: str, + *, + workspace_root: str, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=workspace_root, + permission_mode=permission_mode, + event_sink=event_sink, + ) + + async def resume_task_scoped_stream( + self, + task: str, + *, + resume_session_id: str, + workspace_root: str, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort, + model: str | None = None, + timeout_seconds: float = 300.0, + ) -> AgentEngineResult: + return await self._run_task( + task=task, + model=model, + timeout_seconds=timeout_seconds, + workspace_root=workspace_root, + permission_mode=permission_mode, + event_sink=event_sink, + resume_session_id=resume_session_id, + ) + + async def _run_task( + self, + *, + task: str, + model: str | None, + timeout_seconds: float, + workspace_root: str | None, + permission_mode: PermissionMode, + event_sink: AgentEngineEventSinkPort | None = None, + resume_session_id: str | None = None, ) -> AgentEngineResult: if not self._enabled: return AgentEngineResult( @@ -41,42 +120,80 @@ async def run_task( error="Codex CLI driver is disabled", ) - cmd = [self._cli_path, "exec", "--json", "--full-auto", task] + sandbox = self._sandbox_for(permission_mode) + if sandbox is None: + return AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=False, + output="", + error=( + "Codex non-interactive mode cannot preserve " + "confirm-destructive approvals; use read-only, workspace-write, " + "or danger-full-access" + ), + ) + + cmd = [self._cli_path, "exec", "--json", "--sandbox", sandbox] + if workspace_root: + cmd.extend(["--cd", workspace_root]) if model: cmd.extend(["--model", model]) + if resume_session_id: + cmd.extend(["resume", resume_session_id]) + cmd.append(task) start = time.monotonic() - cli_result = await self._run_cli(cmd, timeout=timeout_seconds) + if event_sink is None: + cli_result = await self._run_cli(cmd, timeout=timeout_seconds) + else: + decoder = CodexJsonlEventDecoder() + + async def publish_line(line: str) -> None: + event = decoder.decode(line) + if event is not None: + await event_sink.publish(event) + + cli_result = await self._run_cli_streaming( + cmd, + timeout=timeout_seconds, + on_stdout_line=publish_line, + ) duration = time.monotonic() - start - if cli_result.returncode != 0: + parsed = parse_codex_output(cli_result.stdout) + metadata: dict = { + "events": [event.model_dump(mode="json") for event in parsed.events], + } + if parsed.session_id: + metadata["session_id"] = parsed.session_id + if parsed.usage: + metadata["usage"] = parsed.usage + if parsed.parse_errors: + metadata["parse_errors"] = parsed.parse_errors + + if cli_result.returncode != 0 or parsed.error: return AgentEngineResult( engine=AgentEngineType.CODEX_CLI, success=False, - output=cli_result.stdout, - error=cli_result.stderr or f"Exit code {cli_result.returncode}", + output=parsed.output, + error=( + parsed.error + or cli_result.stderr + or f"Exit code {cli_result.returncode}" + ), duration_seconds=duration, + model_used=parsed.model or model, + metadata=metadata, ) - output_text = cli_result.stdout - metadata: dict = {} - model_used: str | None = model - try: - data = json.loads(cli_result.stdout) - output_text = data.get("result", cli_result.stdout) - if "model" in data: - model_used = data["model"] - if "usage" in data: - metadata["usage"] = data["usage"] - except (json.JSONDecodeError, TypeError): - pass + model_used = parsed.model or model cost_usd = EngineCostCalculator.calculate(model_used, metadata.get("usage")) return AgentEngineResult( engine=AgentEngineType.CODEX_CLI, success=True, - output=output_text, + output=parsed.output, cost_usd=cost_usd, duration_seconds=duration, model_used=model_used, @@ -93,6 +210,15 @@ def get_capabilities(self) -> AgentEngineCapabilities: supports_sandbox=True, supports_parallel=True, supports_mcp=True, - supports_streaming=False, + supports_streaming=True, cost_per_hour_usd=2.0, ) + + def _sandbox_for(self, permission_mode: PermissionMode | None) -> str | None: + if permission_mode is None or permission_mode is PermissionMode.READ_ONLY: + return "read-only" + if permission_mode is PermissionMode.WORKSPACE_WRITE: + return "workspace-write" + if permission_mode is PermissionMode.DANGER_FULL_ACCESS: + return "danger-full-access" + return None diff --git a/infrastructure/agent_cli/codex_jsonl.py b/infrastructure/agent_cli/codex_jsonl.py new file mode 100644 index 0000000..4edb646 --- /dev/null +++ b/infrastructure/agent_cli/codex_jsonl.py @@ -0,0 +1,194 @@ +"""Parse Codex CLI JSONL into Morphic native-engine events.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any + +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType +from domain.value_objects.agent_engine import AgentEngineType + + +@dataclass(frozen=True) +class CodexParsedRun: + output: str + events: list[AgentEngineEvent] = field(default_factory=list) + session_id: str | None = None + usage: dict[str, int] | None = None + model: str | None = None + error: str | None = None + parse_errors: int = 0 + + +class CodexJsonlEventDecoder: + """Stateful decoder for incremental Codex JSONL event delivery.""" + + def __init__(self) -> None: + self._sequence = 0 + self._session_id: str | None = None + + def decode(self, line: str) -> AgentEngineEvent | None: + try: + raw = json.loads(line) + except json.JSONDecodeError: + return None + if not isinstance(raw, dict) or "type" not in raw: + return None + if raw.get("type") == "thread.started": + self._session_id = _optional_string(raw.get("thread_id")) + event = _normalize_event( + raw, + sequence=self._sequence, + session_id=self._session_id, + ) + self._sequence += 1 + return event + + +def parse_codex_output(stdout: str) -> CodexParsedRun: + """Parse current JSONL output while preserving legacy single-JSON compatibility.""" + + stripped = stdout.strip() + if not stripped: + return CodexParsedRun(output="") + + lines = [line for line in stripped.splitlines() if line.strip()] + decoded: list[dict[str, Any]] = [] + parse_errors = 0 + for line in lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + parse_errors += 1 + continue + if isinstance(value, dict): + decoded.append(value) + else: + parse_errors += 1 + + if len(decoded) == 1 and "type" not in decoded[0]: + legacy = decoded[0] + return CodexParsedRun( + output=str(legacy.get("result", stdout)), + usage=_usage_dict(legacy.get("usage")), + model=_optional_string(legacy.get("model")), + parse_errors=parse_errors, + ) + if not decoded: + return CodexParsedRun(output=stdout, parse_errors=parse_errors) + + events: list[AgentEngineEvent] = [] + session_id: str | None = None + final_message: str | None = None + usage: dict[str, int] | None = None + error: str | None = None + for raw in decoded: + raw_type = str(raw.get("type", "")) + if raw_type == "thread.started": + session_id = _optional_string(raw.get("thread_id")) + if raw_type == "turn.completed": + usage = _usage_dict(raw.get("usage")) + if raw_type in {"turn.failed", "error"}: + error = _error_text(raw) + + item = raw.get("item") if isinstance(raw.get("item"), dict) else {} + if item.get("type") == "agent_message" and raw_type == "item.completed": + final_message = _optional_string(item.get("text")) or final_message + + events.append( + _normalize_event(raw, sequence=len(events), session_id=session_id) + ) + + return CodexParsedRun( + output=final_message or stdout, + events=events, + session_id=session_id, + usage=usage, + error=error, + parse_errors=parse_errors, + ) + + +def _normalize_event( + raw: dict[str, Any], + *, + sequence: int, + session_id: str | None, +) -> AgentEngineEvent: + raw_type = str(raw.get("type", "")) + item = raw.get("item") if isinstance(raw.get("item"), dict) else {} + item_type = _optional_string(item.get("type")) + return AgentEngineEvent( + type=_event_type(raw_type, item_type), + engine=AgentEngineType.CODEX_CLI, + sequence=sequence, + session_id=session_id, + item_id=_optional_string(item.get("id")), + item_type=item_type, + text=_event_text(item, raw), + payload=raw, + ) + + +def _event_type(raw_type: str, item_type: str | None) -> AgentEngineEventType: + if raw_type == "thread.started": + return AgentEngineEventType.RUN_STARTED + if raw_type == "turn.started": + return AgentEngineEventType.TURN_STARTED + if raw_type == "turn.completed": + return AgentEngineEventType.RUN_COMPLETED + if raw_type == "turn.failed": + return AgentEngineEventType.RUN_FAILED + if raw_type == "error": + return AgentEngineEventType.ERROR + if raw_type in {"item.started", "item.completed"}: + if item_type == "agent_message": + return AgentEngineEventType.ASSISTANT_MESSAGE + if item_type == "file_change": + return AgentEngineEventType.FILE_CHANGED + if item_type in {"plan_update", "todo_list"}: + return AgentEngineEventType.PLAN_UPDATED + if item_type in {"command_execution", "mcp_tool_call", "web_search"}: + return ( + AgentEngineEventType.TOOL_STARTED + if raw_type == "item.started" + else AgentEngineEventType.TOOL_COMPLETED + ) + return AgentEngineEventType.PROGRESS + return AgentEngineEventType.UNKNOWN + + +def _event_text(item: dict[str, Any], raw: dict[str, Any]) -> str | None: + for value in [ + item.get("text"), + item.get("command"), + item.get("name"), + raw.get("message"), + raw.get("error"), + ]: + text = _optional_string(value) + if text: + return text + return None + + +def _error_text(raw: dict[str, Any]) -> str: + error = raw.get("error") + if isinstance(error, dict): + return str(error.get("message") or error) + return str(error or raw.get("message") or "Codex turn failed") + + +def _usage_dict(value: object) -> dict[str, int] | None: + if not isinstance(value, dict): + return None + return { + str(key): int(token_count) + for key, token_count in value.items() + if isinstance(token_count, int) and not isinstance(token_count, bool) + } + + +def _optional_string(value: object) -> str | None: + return value if isinstance(value, str) and value else None diff --git a/infrastructure/council/route_chat_direct_runtime.py b/infrastructure/council/route_chat_direct_runtime.py new file mode 100644 index 0000000..c0c83bf --- /dev/null +++ b/infrastructure/council/route_chat_direct_runtime.py @@ -0,0 +1,173 @@ +"""Single-engine route-backed runtime for Morphic Chat CLI.""" + +from __future__ import annotations + +from typing import Protocol + +from pydantic import ValidationError + +from domain.entities.agent_engine_event import AgentEngineEvent +from domain.entities.chat_session import ChatSession, PermissionMode +from domain.entities.council_runtime import CouncilDecision, CouncilRole, CouncilTurn +from domain.entities.workspace_context import ContextIndex +from domain.ports.agent_engine import AgentEngineEventSinkPort, AgentEngineResult +from domain.ports.council_runtime import StreamingCouncilRuntimePort +from domain.value_objects.agent_engine import AgentEngineType +from domain.value_objects.model_tier import TaskType + + +class _RouteExecutor(Protocol): + async def execute( + self, + *, + task: str, + task_type: TaskType = TaskType.SIMPLE_QA, + budget: float = 1.0, + estimated_hours: float = 0.0, + context_tokens: int = 0, + preferred_engine: AgentEngineType | None = None, + timeout_seconds: float = 300.0, + context: str | None = None, + workspace_root: str | None = None, + permission_mode: PermissionMode | None = None, + event_sink: AgentEngineEventSinkPort | None = None, + resume_session_id: str | None = None, + resume_engine: AgentEngineType | None = None, + ) -> AgentEngineResult: ... + + +class RouteChatDirectRuntime(StreamingCouncilRuntimePort): + """Delegate one chat turn to one routed native agent engine.""" + + def __init__( + self, + route_to_engine: _RouteExecutor, + *, + preferred_engine: AgentEngineType | None = None, + budget: float = 1.0, + timeout_seconds: float = 300.0, + ) -> None: + self._route_to_engine = route_to_engine + if preferred_engine not in { + AgentEngineType.CODEX_CLI, + AgentEngineType.CLAUDE_CODE, + }: + raise ValueError( + "direct route requires an explicit streaming native engine: " + "codex_cli or claude_code" + ) + self._preferred_engine = preferred_engine + self._budget = budget + self._timeout_seconds = timeout_seconds + + async def deliberate( + self, + session: ChatSession, + context: ContextIndex, + user_message: str, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + return await self._deliberate( + session=session, + context=context, + user_message=user_message, + event_sink=None, + ) + + async def deliberate_stream( + self, + session: ChatSession, + context: ContextIndex, + user_message: str, + event_sink: AgentEngineEventSinkPort, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + return await self._deliberate( + session=session, + context=context, + user_message=user_message, + event_sink=event_sink, + ) + + async def _deliberate( + self, + *, + session: ChatSession, + context: ContextIndex, + user_message: str, + event_sink: AgentEngineEventSinkPort | None, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + if session.permission_mode is PermissionMode.CONFIRM_DESTRUCTIVE: + raise PermissionError( + "Non-interactive direct route cannot preserve " + "confirm-destructive approvals" + ) + + native_session = session.native_sessions.get(self._preferred_engine.value) + if native_session is not None: + if native_session.workspace_root != context.workspace_root: + raise PermissionError("native session workspace does not match current workspace") + if native_session.permission_mode is not session.permission_mode: + raise PermissionError("native session permission mode does not match current mode") + + result = await self._route_to_engine.execute( + task=user_message, + task_type=TaskType.CODE_GENERATION, + budget=self._budget, + context_tokens=self._context_tokens(context), + preferred_engine=self._preferred_engine, + timeout_seconds=self._timeout_seconds, + context=self._context_summary(context), + workspace_root=context.workspace_root, + permission_mode=session.permission_mode, + event_sink=event_sink, + resume_session_id=( + native_session.session_id if native_session is not None else None + ), + resume_engine=( + native_session.engine if native_session is not None else None + ), + ) + if not result.success: + raise RuntimeError(f"direct route failed: {result.error or 'unknown error'}") + if not result.output.strip(): + raise RuntimeError("direct route returned no output") + + turn = CouncilTurn( + role=CouncilRole.IMPLEMENTER, + engine_id=result.engine.value, + content=result.output.strip(), + evidence=[ + f"route_engine={result.engine.value}", + f"context_sources={len(context.sources)}", + f"permission={session.permission_mode.value}", + ], + engine_events=self._engine_events(result), + cost_usd=result.cost_usd, + latency_ms=int(result.duration_seconds * 1000), + ) + decision = CouncilDecision( + leader_engine_id=turn.engine_id, + selected_role=CouncilRole.IMPLEMENTER, + selected_content=turn.content, + rationale="Single-engine direct route completed the native agent task.", + evidence=turn.evidence, + ) + return [turn], decision + + def _engine_events(self, result: AgentEngineResult) -> list[AgentEngineEvent]: + raw_events = result.metadata.get("events") + if not isinstance(raw_events, list): + return [] + events: list[AgentEngineEvent] = [] + for raw_event in raw_events: + try: + events.append(AgentEngineEvent.model_validate(raw_event, strict=False)) + except ValidationError: + continue + return events + + def _context_tokens(self, context: ContextIndex) -> int: + return sum(len(source.sections) for source in context.sources) + + def _context_summary(self, context: ContextIndex) -> str: + sources = ", ".join(source.source_path for source in context.sources[:10]) + return f"context_sources={len(context.sources)}; sources={sources}" diff --git a/infrastructure/hooks/noop_hook_executor.py b/infrastructure/hooks/noop_hook_executor.py new file mode 100644 index 0000000..9483f26 --- /dev/null +++ b/infrastructure/hooks/noop_hook_executor.py @@ -0,0 +1,19 @@ +"""No-op implementation of the chat hook executor port.""" + +from __future__ import annotations + +from domain.entities.hook import HookExecutionRequest, HookExecutionResult +from domain.ports.hook_executor import HookExecutorPort + + +class NoopHookExecutor(HookExecutorPort): + """Records hook execution intent without invoking a shell command.""" + + async def execute(self, request: HookExecutionRequest) -> HookExecutionResult: + return HookExecutionResult( + request_id=request.id, + success=True, + stdout_summary=f"Hook {request.hook_name} not executed by no-op executor.", + stderr_summary="", + exit_code=0, + ) diff --git a/infrastructure/hooks/shell_hook_executor.py b/infrastructure/hooks/shell_hook_executor.py new file mode 100644 index 0000000..c8969c1 --- /dev/null +++ b/infrastructure/hooks/shell_hook_executor.py @@ -0,0 +1,52 @@ +"""Shell-backed implementation of the chat hook executor port.""" + +from __future__ import annotations + +from pathlib import Path + +from domain.entities.execution import Action +from domain.entities.hook import HookExecutionRequest, HookExecutionResult +from domain.ports.hook_executor import HookExecutorPort +from domain.ports.local_executor import LocalExecutorPort +from domain.value_objects import RiskLevel +from domain.value_objects.status import ObservationStatus + + +class ShellHookExecutor(HookExecutorPort): + """Execute hook commands through LAEE shell execution policy.""" + + def __init__( + self, + *, + local_executor: LocalExecutorPort, + workspace_root: str | Path, + timeout_seconds: int = 30, + ) -> None: + self._local_executor = local_executor + self._workspace_root = Path(workspace_root) + self._timeout_seconds = timeout_seconds + + async def execute(self, request: HookExecutionRequest) -> HookExecutionResult: + observation = await self._local_executor.execute( + Action( + tool="shell_exec", + args={ + "cmd": request.command, + "cwd": str(self._workspace_root), + "timeout": self._timeout_seconds, + }, + description=( + f"chat session {request.session_id} hook {request.hook_name} " + f"from {request.source_path}" + ), + risk=RiskLevel.MEDIUM, + ) + ) + success = observation.status is ObservationStatus.SUCCESS + return HookExecutionResult( + request_id=request.id, + success=success, + stdout_summary=observation.result if success else "", + stderr_summary="" if success else observation.result, + exit_code=0 if success else 1, + ) diff --git a/infrastructure/tools/noop_tool_executor.py b/infrastructure/tools/noop_tool_executor.py new file mode 100644 index 0000000..bb9d10a --- /dev/null +++ b/infrastructure/tools/noop_tool_executor.py @@ -0,0 +1,22 @@ +"""No-op implementation of the chat tool executor port.""" + +from __future__ import annotations + +from domain.ports.tool_executor import ( + ToolExecutionRequest, + ToolExecutionResult, + ToolExecutorPort, +) + + +class NoopToolExecutor(ToolExecutorPort): + """Records tool execution intent without invoking local tools.""" + + async def execute(self, request: ToolExecutionRequest) -> ToolExecutionResult: + return ToolExecutionResult( + request_id=request.id, + success=True, + stdout_summary=f"Tool {request.tool_name} not executed by no-op executor.", + stderr_summary="", + exit_code=0, + ) diff --git a/interface/cli/chat_command.py b/interface/cli/chat_command.py index d794816..3fcd4d1 100644 --- a/interface/cli/chat_command.py +++ b/interface/cli/chat_command.py @@ -5,7 +5,7 @@ import json import logging import os -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from pathlib import Path @@ -15,11 +15,19 @@ from domain.entities.council_runtime import CouncilRole from domain.ports.council_runtime import CouncilRuntimePort from domain.ports.engine_registry import EngineRegistryPort +from domain.ports.hook_executor import HookExecutorPort +from domain.ports.local_executor import LocalExecutorPort +from domain.ports.tool_executor import ToolExecutorPort from domain.value_objects.agent_engine import AgentEngineType from infrastructure.council.local_chat_council_runtime import LocalChatCouncilRuntime from infrastructure.council.route_chat_council_runtime import RouteChatCouncilRuntime +from infrastructure.council.route_chat_direct_runtime import RouteChatDirectRuntime from infrastructure.engines.route_engine_registry import RouteEngineRegistry from infrastructure.engines.static_engine_registry import StaticEngineRegistry +from infrastructure.hooks.noop_hook_executor import NoopHookExecutor +from infrastructure.hooks.shell_hook_executor import ShellHookExecutor +from infrastructure.tools.laee_tool_executor import LaeeToolExecutor +from infrastructure.tools.noop_tool_executor import NoopToolExecutor from interface.cli._utils import _get_container, _run from interface.cli.chat_repl import ChatRepl from interface.cli.formatters import console @@ -44,6 +52,21 @@ "--route-council", help="Use route-backed engines for chat council roles.", ) +_CHAT_ROUTE_DIRECT_OPTION = typer.Option( + False, + "--route-direct", + help="Use one route-backed native engine for each chat turn.", +) +_CODE_ROUTE_DIRECT_OPTION = typer.Option( + False, + "--route-direct", + help="Use one route-backed native engine for the coding goal.", +) +_DIRECT_ENGINE_OPTION = typer.Option( + None, + "--engine", + help="Preferred engine for --route-direct; omit for automatic routing.", +) _PLANNER_ENGINE_OPTION = typer.Option( None, "--planner-engine", @@ -59,6 +82,16 @@ "--leader-engine", help="Preferred route engine for the leader role.", ) +_PERMISSION_MODE_OPTION = typer.Option( + PermissionMode.CONFIRM_DESTRUCTIVE, + "--permission-mode", + help="Workspace permission mode.", +) +_BENCHMARK_RECEIPT_OPTION = typer.Option( + False, + "--benchmark-receipt", + help="Emit a canonical Morphic benchmark receipt as the final stdout line.", +) @contextmanager @@ -91,17 +124,29 @@ def chat_cmd( "--json", help="Emit machine-readable JSON for diagnostics.", ), + control: bool = typer.Option( + False, + "--control", + help="Enable authenticated loopback control for active turns.", + ), route_council: bool = _CHAT_ROUTE_COUNCIL_OPTION, + route_direct: bool = _CHAT_ROUTE_DIRECT_OPTION, + direct_engine: str | None = _DIRECT_ENGINE_OPTION, planner_engine: str | None = _PLANNER_ENGINE_OPTION, critic_engine: str | None = _CRITIC_ENGINE_OPTION, leader_engine: str | None = _LEADER_ENGINE_OPTION, + permission_mode: PermissionMode = _PERMISSION_MODE_OPTION, workspace: Path | None = _CHAT_WORKSPACE_OPTION, ) -> None: """Start the Morphic terminal chat REPL.""" if doctor: - with _disabled_logging(json_output): - engine_registry = _chat_engine_registry() - payload = _run(_chat_doctor_payload(engine_registry=engine_registry)) + try: + with _disabled_logging(json_output): + engine_registry = _chat_engine_registry() + payload = _run(_chat_doctor_payload(engine_registry=engine_registry)) + except ValueError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=2) from None if json_output: typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) else: @@ -113,6 +158,8 @@ def chat_cmd( try: council_runtime = _chat_council_runtime( route_council=route_council, + route_direct=route_direct, + direct_engine=direct_engine, planner_engine=planner_engine, critic_engine=critic_engine, leader_engine=leader_engine, @@ -120,21 +167,35 @@ def chat_cmd( except ValueError as exc: typer.echo(f"Error: {exc}", err=True) raise typer.Exit(code=2) from None - _run( - ChatRepl( - workspace_root=workspace or Path.cwd(), - council_runtime=council_runtime, - engine_registry=engine_registry, - ).run(resume=resume) - ) + try: + _run( + ChatRepl( + workspace_root=workspace or Path.cwd(), + council_runtime=council_runtime, + engine_registry=engine_registry, + hook_executor_factory=lambda root: _chat_hook_executor(workspace_root=root), + tool_executor_factory=lambda _root: _chat_tool_executor(), + control_enabled=control, + ).run(resume=resume, permission_mode=permission_mode) + ) + except (PermissionError, RuntimeError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from None + except KeyboardInterrupt: + typer.echo("Cancelled.", err=True) + raise typer.Exit(code=130) from None def code_cmd( goal: str = typer.Argument(..., help="One-shot coding goal."), route_council: bool = _CODE_ROUTE_COUNCIL_OPTION, + route_direct: bool = _CODE_ROUTE_DIRECT_OPTION, + direct_engine: str | None = _DIRECT_ENGINE_OPTION, planner_engine: str | None = _PLANNER_ENGINE_OPTION, critic_engine: str | None = _CRITIC_ENGINE_OPTION, leader_engine: str | None = _LEADER_ENGINE_OPTION, + permission_mode: PermissionMode = _PERMISSION_MODE_OPTION, + benchmark_receipt: bool = _BENCHMARK_RECEIPT_OPTION, workspace: Path | None = _CODE_WORKSPACE_OPTION, ) -> None: """Run one coding goal and persist the session ledger.""" @@ -143,6 +204,8 @@ def code_cmd( try: council_runtime = _chat_council_runtime( route_council=route_council, + route_direct=route_direct, + direct_engine=direct_engine, planner_engine=planner_engine, critic_engine=critic_engine, leader_engine=leader_engine, @@ -150,13 +213,26 @@ def code_cmd( except ValueError as exc: typer.echo(f"Error: {exc}", err=True) raise typer.Exit(code=2) from None - _run( - ChatRepl( - workspace_root=workspace or Path.cwd(), - council_runtime=council_runtime, - engine_registry=engine_registry, - ).run_goal(goal=goal) - ) + try: + result = _run( + ChatRepl( + workspace_root=workspace or Path.cwd(), + council_runtime=council_runtime, + engine_registry=engine_registry, + hook_executor_factory=lambda root: _chat_hook_executor(workspace_root=root), + tool_executor_factory=lambda _root: _chat_tool_executor(), + ).run_goal_with_result(goal=goal, permission_mode=permission_mode) + ) + except (PermissionError, RuntimeError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from None + except KeyboardInterrupt: + typer.echo("Cancelled.", err=True) + raise typer.Exit(code=130) from None + if benchmark_receipt: + from benchmarks.agent_cli_receipts import build_morphic_benchmark_receipt + + typer.echo(build_morphic_benchmark_receipt(result.turns).to_json()) def _chat_engine_registry() -> EngineRegistryPort: @@ -170,18 +246,125 @@ def _chat_engine_registry() -> EngineRegistryPort: return StaticEngineRegistry() +def _chat_hook_executor( + *, + workspace_root: Path, + local_executor_factory: Callable[[], LocalExecutorPort] | None = None, +) -> HookExecutorPort: + mode = _chat_hook_execution_mode() + if mode == "noop": + return NoopHookExecutor() + factory = local_executor_factory or _chat_local_executor + return ShellHookExecutor( + local_executor=factory(), + workspace_root=workspace_root, + ) + + +def _chat_hook_execution_mode() -> str: + mode = os.getenv("MORPHIC_CHAT_HOOK_EXECUTION", "noop").strip().lower() + if mode == "": + return "noop" + if mode in {"noop", "shell"}: + return mode + raise ValueError( + "Invalid hook execution mode " + f"'{mode}'. Expected one of: noop, shell" + ) + + +def _chat_tool_executor( + *, + local_executor_factory: Callable[[], LocalExecutorPort] | None = None, +) -> ToolExecutorPort: + mode = _chat_tool_execution_mode() + if mode == "noop": + return NoopToolExecutor() + factory = local_executor_factory or _chat_local_executor + return LaeeToolExecutor(local_executor=factory()) + + +def _chat_tool_execution_mode() -> str: + mode = os.getenv("MORPHIC_CHAT_TOOL_EXECUTION", "noop").strip().lower() + if mode == "": + return "noop" + if mode in {"noop", "laee"}: + return mode + raise ValueError( + "Invalid tool execution mode " + f"'{mode}'. Expected one of: noop, laee" + ) + + +def _chat_local_executor() -> LocalExecutorPort: + from domain.value_objects.approval_mode import ApprovalMode + from infrastructure.local_execution.audit_log import JsonlAuditLogger + from infrastructure.local_execution.executor import LocalExecutor + + try: + settings = _get_container().settings + approval_mode_value = settings.laee_approval_mode + audit_log_path = settings.laee_audit_log_path + undo_enabled = settings.laee_undo_enabled + except Exception: + approval_mode_value = "confirm-destructive" + audit_log_path = Path(".morphic/audit_log.jsonl") + undo_enabled = True + + mode_map = { + "full-auto": ApprovalMode.FULL_AUTO, + "confirm-destructive": ApprovalMode.CONFIRM_DESTRUCTIVE, + "confirm-all": ApprovalMode.CONFIRM_ALL, + } + approval_mode = mode_map.get( + approval_mode_value, + ApprovalMode.CONFIRM_DESTRUCTIVE, + ) + return LocalExecutor( + approval_mode=approval_mode, + audit_logger=JsonlAuditLogger(log_path=audit_log_path), + undo_enabled=undo_enabled, + ) + + def _chat_council_runtime( *, route_council: bool = False, + route_direct: bool = False, + direct_engine: str | None = None, planner_engine: str | None = None, critic_engine: str | None = None, leader_engine: str | None = None, ) -> CouncilRuntimePort: + if route_direct and route_council: + raise ValueError("--route-direct and --route-council are mutually exclusive") + preferred_direct_engine = _direct_engine_preference(direct_engine) + if preferred_direct_engine is not None and not route_direct: + raise ValueError("--engine requires --route-direct") role_engines = _role_engine_preferences( planner_engine=planner_engine, critic_engine=critic_engine, leader_engine=leader_engine, ) + if route_direct: + if preferred_direct_engine not in { + AgentEngineType.CODEX_CLI, + AgentEngineType.CLAUDE_CODE, + }: + raise ValueError( + "--route-direct requires --engine codex_cli or claude_code" + ) + try: + container = _get_container() + route_to_engine = getattr(container, "route_to_engine", None) + if route_to_engine is not None: + return RouteChatDirectRuntime( + route_to_engine, + preferred_engine=preferred_direct_engine, + ) + except Exception as exc: + raise ValueError(f"Direct route is unavailable: {exc}") from exc + raise ValueError("Direct route is unavailable: route engine is not configured") if not route_council and os.getenv("MORPHIC_CHAT_ROUTE_COUNCIL") != "1": return LocalChatCouncilRuntime() try: @@ -197,6 +380,18 @@ def _chat_council_runtime( return LocalChatCouncilRuntime() +def _direct_engine_preference(engine_id: str | None) -> AgentEngineType | None: + if engine_id is None: + return None + try: + return AgentEngineType(engine_id) + except ValueError as exc: + valid = ", ".join(engine.value for engine in AgentEngineType) + raise ValueError( + f"Invalid direct engine '{engine_id}'. Expected one of: {valid}" + ) from exc + + def _role_engine_preferences( *, planner_engine: str | None, @@ -228,5 +423,7 @@ async def _chat_doctor_payload( engines = await registry.list_engines() return { "engines": [engine.model_dump(mode="json") for engine in engines], + "hook_execution_mode": _chat_hook_execution_mode(), "permission_modes": [mode.value for mode in PermissionMode], + "tool_execution_mode": _chat_tool_execution_mode(), } diff --git a/interface/cli/chat_control_transport.py b/interface/cli/chat_control_transport.py new file mode 100644 index 0000000..5a62d9c --- /dev/null +++ b/interface/cli/chat_control_transport.py @@ -0,0 +1,312 @@ +"""Authenticated loopback transport for controlling one active chat turn.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import secrets +from contextlib import suppress +from pathlib import Path +from typing import Any, Self + +from interface.cli.turn_control import ActiveTurnController + +_HOST = "127.0.0.1" +_MAX_LINE_BYTES = 4096 +_MAX_STEER_PROMPT_BYTES = 2048 +_PROTOCOL_VERSION = 1 +_REQUEST_TIMEOUT_SECONDS = 2.0 +_SUPPORTED_COMMANDS = frozenset({"cancel", "status", "steer"}) + + +class ChatControlServer: + """Expose one controller through a short-lived authenticated loopback server.""" + + def __init__( + self, + *, + workspace_root: Path, + session_id: str, + turn_controller: ActiveTurnController, + ) -> None: + self._workspace_root = workspace_root + self._session_id = session_id + self._turn_controller = turn_controller + self._token = secrets.token_urlsafe(32) + self._server: asyncio.Server | None = None + self._port: int | None = None + + @property + def descriptor_path(self) -> Path: + return _descriptor_path(self._workspace_root, self._session_id) + + async def __aenter__(self) -> Self: + await self.start() + return self + + async def __aexit__(self, *_exc_info: object) -> None: + await self.close() + + async def start(self) -> None: + if self._server is not None: + raise RuntimeError("chat control server is already running") + + server = await asyncio.start_server( + self._handle_client, + host=_HOST, + port=0, + limit=_MAX_LINE_BYTES, + ) + sockets = server.sockets + if not sockets: + server.close() + await server.wait_closed() + raise RuntimeError("chat control server did not bind a socket") + + self._server = server + self._port = int(sockets[0].getsockname()[1]) + try: + self._write_descriptor() + except Exception: + await self.close() + raise + + async def close(self) -> None: + server = self._server + self._server = None + self._port = None + if server is not None: + server.close() + await server.wait_closed() + self._remove_owned_descriptor() + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + try: + raw_request = await asyncio.wait_for( + reader.readline(), + timeout=_REQUEST_TIMEOUT_SECONDS, + ) + response = self._decode_and_dispatch(raw_request) + except TimeoutError: + response = {"error": "request_timeout", "ok": False} + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + response = {"error": "invalid_request", "ok": False} + + writer.write( + json.dumps(response, ensure_ascii=False, sort_keys=True).encode("utf-8") + + b"\n" + ) + with suppress(ConnectionError): + await writer.drain() + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + + def _decode_and_dispatch(self, raw_request: bytes) -> dict[str, object]: + if not raw_request or len(raw_request) > _MAX_LINE_BYTES: + raise ValueError("invalid control request size") + request = json.loads(raw_request.decode("utf-8")) + if not isinstance(request, dict): + raise ValueError("control request must be an object") + + token = request.get("token") + if not isinstance(token, str) or not secrets.compare_digest(token, self._token): + return {"error": "unauthorized", "ok": False} + if request.get("session_id") != self._session_id: + return {"error": "session_mismatch", "ok": False} + + command = request.get("command") + if command not in _SUPPORTED_COMMANDS: + return {"error": "unsupported_command", "ok": False} + + active_turn = self._turn_controller.has_active_turn + response: dict[str, object] = { + "active_turn": active_turn, + "ok": True, + "session_id": self._session_id, + } + if command == "cancel": + response["cancelled"] = self._turn_controller.cancel_active_turn() + elif command == "steer": + try: + prompt = _normalize_steer_prompt(request.get("prompt")) + except ValueError: + return {"error": "invalid_steer_prompt", "ok": False} + response["steered"] = self._turn_controller.steer_active_turn(prompt) + return response + + def _write_descriptor(self) -> None: + if self._port is None: + raise RuntimeError("chat control server is not bound") + path = self.descriptor_path + path.parent.mkdir(parents=True, exist_ok=True) + path.parent.chmod(0o700) + payload = { + "host": _HOST, + "port": self._port, + "protocol_version": _PROTOCOL_VERSION, + "session_id": self._session_id, + "token": self._token, + } + temporary = path.with_suffix(f".{self._token[:12]}.tmp") + try: + temporary.write_text( + json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary.chmod(0o600) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + path.chmod(0o600) + + def _remove_owned_descriptor(self) -> None: + path = self.descriptor_path + try: + descriptor = _read_descriptor(path) + except (OSError, ValueError, json.JSONDecodeError): + return + if descriptor.get("token") == self._token: + path.unlink(missing_ok=True) + + +async def send_chat_control_command( + *, + workspace_root: Path, + session_id: str, + command: str, + prompt: str | None = None, +) -> dict[str, object]: + """Send one authenticated command or report an inactive missing descriptor.""" + if command not in _SUPPORTED_COMMANDS: + raise ValueError(f"unsupported chat control command: {command}") + normalized_prompt: str | None = None + if command == "steer": + normalized_prompt = _normalize_steer_prompt(prompt) + elif prompt is not None: + raise ValueError("prompt is supported only for the steer command") + + descriptor_path = _descriptor_path(workspace_root, session_id) + if not descriptor_path.exists(): + response: dict[str, object] = { + "active_turn": False, + "ok": True, + "session_id": session_id, + } + if command == "cancel": + response["cancelled"] = False + elif command == "steer": + response["steered"] = False + return response + + descriptor = _validated_descriptor(descriptor_path, session_id=session_id) + request = { + "command": command, + "session_id": session_id, + "token": descriptor["token"], + } + if normalized_prompt is not None: + request["prompt"] = normalized_prompt + encoded_request = ( + json.dumps(request, ensure_ascii=False, sort_keys=True).encode("utf-8") + b"\n" + ) + if len(encoded_request) > _MAX_LINE_BYTES: + raise ValueError("steer prompt exceeds the control request limit") + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection( + host=str(descriptor["host"]), + port=int(descriptor["port"]), + limit=_MAX_LINE_BYTES, + ), + timeout=_REQUEST_TIMEOUT_SECONDS, + ) + except (OSError, TimeoutError) as exc: + raise RuntimeError("chat control endpoint is unavailable") from exc + + try: + writer.write(encoded_request) + await writer.drain() + raw_response = await asyncio.wait_for( + reader.readline(), + timeout=_REQUEST_TIMEOUT_SECONDS, + ) + finally: + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + + response = json.loads(raw_response.decode("utf-8")) + if not isinstance(response, dict) or not isinstance(response.get("ok"), bool): + raise RuntimeError("chat control endpoint returned an invalid response") + if response["ok"] is not True: + error = response.get("error", "unknown_error") + raise RuntimeError(f"chat control request rejected: {error}") + return response + + +def discover_active_chat_sessions(*, workspace_root: Path) -> list[str]: + """List session ids with syntactically valid local control descriptors.""" + control_dir = workspace_root / ".morphic" / "control" + if not control_dir.exists(): + return [] + sessions: list[str] = [] + for path in sorted(control_dir.glob("*.json")): + try: + descriptor = _validated_descriptor(path) + except (OSError, ValueError, json.JSONDecodeError): + continue + sessions.append(str(descriptor["session_id"])) + return sessions + + +def _descriptor_path(workspace_root: Path, session_id: str) -> Path: + digest = hashlib.sha256(session_id.encode("utf-8")).hexdigest()[:32] + return workspace_root / ".morphic" / "control" / f"{digest}.json" + + +def _normalize_steer_prompt(prompt: object) -> str: + if not isinstance(prompt, str): + raise ValueError("steer prompt must be text") + normalized = prompt.strip() + if not normalized: + raise ValueError("steer prompt must not be empty") + if len(normalized.encode("utf-8")) > _MAX_STEER_PROMPT_BYTES: + raise ValueError("steer prompt must not exceed 2048 UTF-8 bytes") + return normalized + + +def _read_descriptor(path: Path) -> dict[str, Any]: + descriptor = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(descriptor, dict): + raise ValueError("chat control descriptor must be an object") + return descriptor + + +def _validated_descriptor( + path: Path, + *, + session_id: str | None = None, +) -> dict[str, Any]: + descriptor = _read_descriptor(path) + if descriptor.get("protocol_version") != _PROTOCOL_VERSION: + raise ValueError("unsupported chat control protocol") + if descriptor.get("host") != _HOST: + raise ValueError("chat control host must be loopback") + port = descriptor.get("port") + if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535: + raise ValueError("invalid chat control port") + token = descriptor.get("token") + if not isinstance(token, str) or not token: + raise ValueError("invalid chat control token") + descriptor_session = descriptor.get("session_id") + if not isinstance(descriptor_session, str) or not descriptor_session: + raise ValueError("invalid chat control session") + if session_id is not None and descriptor_session != session_id: + raise ValueError("chat control session mismatch") + return descriptor diff --git a/interface/cli/chat_repl.py b/interface/cli/chat_repl.py index ca2ee88..a37498d 100644 --- a/interface/cli/chat_repl.py +++ b/interface/cli/chat_repl.py @@ -2,23 +2,51 @@ from __future__ import annotations +import json import uuid +from collections.abc import Callable +from dataclasses import dataclass +from functools import partial from pathlib import Path +from typing import Any from application.use_cases.discover_workspace_context import DiscoverWorkspaceContextUseCase +from application.use_cases.execute_chat_hook import ExecuteChatHookUseCase +from application.use_cases.execute_chat_tool import ExecuteChatToolUseCase from application.use_cases.execute_slash_command import ExecuteSlashCommandUseCase from application.use_cases.resume_chat_session import ResumeChatSessionUseCase from application.use_cases.send_chat_message import SendChatMessageUseCase from application.use_cases.start_chat_session import StartChatSessionUseCase +from domain.entities.chat_event import ChatEventType from domain.entities.chat_session import ChatSession, PermissionMode +from domain.entities.council_runtime import CouncilTurn +from domain.entities.execution import Action +from domain.entities.hook import HookType from domain.entities.workspace_context import ContextIndex +from domain.ports.agent_engine import AgentEngineEventSinkPort from domain.ports.council_runtime import CouncilRuntimePort from domain.ports.engine_registry import EngineRegistryPort +from domain.ports.hook_executor import HookExecutorPort +from domain.ports.tool_executor import ToolExecutorPort +from domain.services.risk_assessor import RiskAssessor from infrastructure.chat.jsonl_session_store import JsonlChatSessionStore from infrastructure.context.workspace_context_discovery import WorkspaceContextDiscovery from infrastructure.council.local_chat_council_runtime import LocalChatCouncilRuntime from infrastructure.engines.static_engine_registry import StaticEngineRegistry +from infrastructure.hooks.noop_hook_executor import NoopHookExecutor +from infrastructure.hooks.workspace_hook_registry import WorkspaceHookRegistry +from infrastructure.tools.noop_tool_executor import NoopToolExecutor +from interface.cli.chat_control_transport import ChatControlServer from interface.cli.formatters import console +from interface.cli.native_event_progress import NativeEventProgressRenderer +from interface.cli.slash_commands import parse_slash_command +from interface.cli.turn_control import ActiveTurnController, TurnCancelledError + + +@dataclass(frozen=True) +class GoalRunResult: + output: str + turns: tuple[CouncilTurn, ...] class ChatRepl: @@ -30,12 +58,25 @@ def __init__( workspace_root: Path, council_runtime: CouncilRuntimePort | None = None, engine_registry: EngineRegistryPort | None = None, + hook_executor_factory: Callable[[Path], HookExecutorPort] | None = None, + tool_executor_factory: Callable[[Path], ToolExecutorPort] | None = None, + engine_event_observer: AgentEngineEventSinkPort | None = None, + turn_controller: ActiveTurnController | None = None, + control_enabled: bool = False, ) -> None: self._workspace_root = workspace_root self._session_store = JsonlChatSessionStore(workspace_root=workspace_root) self._context_discovery = WorkspaceContextDiscovery() self._council_runtime = council_runtime or LocalChatCouncilRuntime() self._engine_registry = engine_registry or StaticEngineRegistry() + self._hook_executor_factory = hook_executor_factory or (lambda _root: NoopHookExecutor()) + self._tool_executor_factory = tool_executor_factory or (lambda _root: NoopToolExecutor()) + self._risk_assessor = RiskAssessor() + self._engine_event_observer = ( + engine_event_observer or NativeEventProgressRenderer() + ) + self._turn_controller = turn_controller or ActiveTurnController() + self._control_enabled = control_enabled async def run( self, @@ -50,15 +91,29 @@ async def run( ) session, context = await self._discover_context(session) console.print(f"Morphic chat session {session.id}") + queued_line: str | None = None while True: - try: - line = input("> ") - except EOFError: - break + is_steered = queued_line is not None + if is_steered: + line = queued_line + queued_line = None + else: + try: + line = input("> ") + except EOFError: + break if not line.strip(): continue - if line.strip().startswith("/"): + if not is_steered and line.strip().startswith("/"): + if line.strip().startswith("/hooks "): + session, output = await self._execute_hooks_command(session, line) + console.print(output) + continue + if line.strip().startswith("/tools "): + session, output = await self._execute_tools_command(session, line) + console.print(output) + continue result = await ExecuteSlashCommandUseCase( session_store=self._session_store, engine_registry=self._engine_registry, @@ -69,10 +124,48 @@ async def run( break continue - result = await SendChatMessageUseCase( - session_store=self._session_store, - council_runtime=self._council_runtime, - ).execute(session=session, context=context, message=line) + try: + send_message = SendChatMessageUseCase( + session_store=self._session_store, + council_runtime=self._council_runtime, + engine_event_observer=self._engine_event_observer, + ).execute + operation = partial( + send_message, + session=session, + context=context, + message=line, + ) + if self._control_enabled: + async with ChatControlServer( + workspace_root=self._workspace_root, + session_id=session.id, + turn_controller=self._turn_controller, + ): + result = await self._turn_controller.run(operation) + else: + result = await self._turn_controller.run(operation) + except TurnCancelledError: + resumed = await ResumeChatSessionUseCase( + session_store=self._session_store, + ).execute(session.id) + session = resumed.session + queued_line = self._turn_controller.take_steer_prompt() + if queued_line is None: + console.print("Turn cancelled.") + else: + session, steered_event = session.record_event( + ChatEventType.TURN_STEERED, + { + "replacement_prompt_bytes": len( + queued_line.encode("utf-8") + ), + "source": "turn_controller", + }, + ) + await self._session_store.append_event(steered_event) + console.print("Turn steered.") + continue session = result.session console.print(result.events[-1].payload["text"]) @@ -84,6 +177,19 @@ async def run_goal( goal: str, permission_mode: PermissionMode = PermissionMode.CONFIRM_DESTRUCTIVE, ) -> str: + return ( + await self.run_goal_with_result( + goal=goal, + permission_mode=permission_mode, + ) + ).output + + async def run_goal_with_result( + self, + *, + goal: str, + permission_mode: PermissionMode = PermissionMode.CONFIRM_DESTRUCTIVE, + ) -> GoalRunResult: session = await self._load_or_start_session( resume=None, permission_mode=permission_mode, @@ -93,10 +199,11 @@ async def run_goal( result = await SendChatMessageUseCase( session_store=self._session_store, council_runtime=self._council_runtime, + engine_event_observer=self._engine_event_observer, ).execute(session=session, context=context, message=goal) output = str(result.events[-1].payload["text"]) console.print(output) - return output + return GoalRunResult(output=output, turns=result.turns) async def _load_or_start_session( self, @@ -126,3 +233,144 @@ async def _discover_context(self, session: ChatSession) -> tuple[ChatSession, Co def _new_session_id(self) -> str: return uuid.uuid4().hex[:12] + + async def _execute_hooks_command( + self, + session: ChatSession, + line: str, + ) -> tuple[ChatSession, str]: + command = parse_slash_command(line) + verb, _, raw_hook_type = command.args.partition(" ") + if verb != "run" or not raw_hook_type: + return session, "usage: /hooks run " + + try: + hook_type = HookType(raw_hook_type) + except ValueError: + valid = ", ".join(item.value for item in HookType) + return session, f"invalid hook type: {raw_hook_type} expected one of: {valid}" + + current, command_event = session.record_event( + ChatEventType.SLASH_COMMAND, + {"command": line}, + ) + await self._session_store.append_event(command_event) + + hook_executor = self._hook_executor_factory(self._workspace_root) + hook_result = await ExecuteChatHookUseCase( + session_store=self._session_store, + hook_registry=WorkspaceHookRegistry(self._workspace_root), + hook_executor=hook_executor, + ).execute(session=current, hook_type=hook_type) + current = hook_result.session + + skipped = sum( + 1 for event in hook_result.events if event.type is ChatEventType.HOOK_EXECUTION_SKIPPED + ) + failed = sum(1 for result in hook_result.hook_results if not result.success) + succeeded = sum(1 for result in hook_result.hook_results if result.success) + mode = "shell" if hook_executor.__class__.__name__ == "ShellHookExecutor" else "noop" + output = ( + f"hooks type={hook_type.value} mode={mode} " + f"succeeded={succeeded} failed={failed} skipped={skipped}" + ) + current, assistant_event = current.record_event( + ChatEventType.ASSISTANT_MESSAGE, + {"text": output, "source": "/hooks"}, + ) + await self._session_store.append_event(assistant_event) + return current, output + + async def _execute_tools_command( + self, + session: ChatSession, + line: str, + ) -> tuple[ChatSession, str]: + command = parse_slash_command(line) + verb, _, rest = command.args.partition(" ") + tool_name, _, raw_arguments = rest.partition(" ") + if verb != "run" or not tool_name: + return session, "usage: /tools run [json_arguments]" + + try: + arguments = self._parse_tool_arguments(raw_arguments) + except ValueError as exc: + return session, str(exc) + + current, command_event = session.record_event( + ChatEventType.SLASH_COMMAND, + {"command": line}, + ) + await self._session_store.append_event(command_event) + + hook_executor = self._hook_executor_factory(self._workspace_root) + hook_runner = ExecuteChatHookUseCase( + session_store=self._session_store, + hook_registry=WorkspaceHookRegistry(self._workspace_root), + hook_executor=hook_executor, + ) + tool_executor = self._tool_executor_factory(self._workspace_root) + try: + result = await ExecuteChatToolUseCase( + session_store=self._session_store, + tool_executor=tool_executor, + hook_runner=hook_runner, + ).execute( + session=current, + tool_name=tool_name, + arguments=arguments, + risk_level=self._risk_assessor.assess( + Action(tool=tool_name, args=arguments) + ), + ) + except PermissionError as exc: + output = f"permission denied: {exc}" + current, assistant_event = current.record_event( + ChatEventType.ASSISTANT_MESSAGE, + {"text": output, "source": "/tools"}, + ) + await self._session_store.append_event(assistant_event) + return current, output + current = result.session + + mode = "laee" if tool_executor.__class__.__name__ == "LaeeToolExecutor" else "noop" + output = self._format_tool_output( + tool_name=tool_name, + mode=mode, + success=result.tool_result.success, + exit_code=result.tool_result.exit_code, + stderr_summary=result.tool_result.stderr_summary, + ) + current, assistant_event = current.record_event( + ChatEventType.ASSISTANT_MESSAGE, + {"text": output, "source": "/tools"}, + ) + await self._session_store.append_event(assistant_event) + return current, output + + def _format_tool_output( + self, + *, + tool_name: str, + mode: str, + success: bool, + exit_code: int | None, + stderr_summary: str, + ) -> str: + output = f"tools tool={tool_name} mode={mode} success={success}" + if exit_code is not None: + output = f"{output} exit_code={exit_code}" + if not success and stderr_summary: + output = f"{output} error={stderr_summary}" + return output + + def _parse_tool_arguments(self, raw_arguments: str) -> dict[str, Any]: + if not raw_arguments: + return {} + try: + parsed = json.loads(raw_arguments) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSON arguments: {exc.msg}") from exc + if not isinstance(parsed, dict): + raise ValueError("invalid JSON arguments: expected object") + return parsed diff --git a/interface/cli/commands/benchmark.py b/interface/cli/commands/benchmark.py index a0b3c26..a16d0aa 100644 --- a/interface/cli/commands/benchmark.py +++ b/interface/cli/commands/benchmark.py @@ -2,6 +2,11 @@ from __future__ import annotations +import json +import os +import sys +import tempfile +from pathlib import Path from typing import Any import typer @@ -9,6 +14,642 @@ from interface.cli.formatters import console benchmark_app = typer.Typer(no_args_is_help=True) +_AGENT_CLI_MANIFEST_OPTION = typer.Option( + ..., + "--manifest", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Same-task benchmark manifest JSON.", +) +_AGENT_CLI_RESULTS_OPTION = typer.Option( + ..., + "--results", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Recorded trial observations JSON.", +) +_AGENT_CLI_JSON_OPTION = typer.Option(False, "--json", help="Emit deterministic JSON.") +_CHECKPOINT_TLS_REVOCATIONS_OPTION = typer.Option( + None, "--revocations", exists=True, file_okay=True, dir_okay=False, readable=True +) +_RECORDER_CONFIG_OPTION = typer.Option( + ..., + "--config", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Recorder command configuration JSON.", +) +_RECORDER_WORKTREE_OPTION = typer.Option(None, "--worktree-root", help="Isolated worktree root.") +_RECORDER_SOURCE_OPTION = typer.Option(None, "--source-root", help="Source Git workspace.") +_RECORDER_EVIDENCE_OPTION = typer.Option(None, "--evidence", help="Evidence JSON output path.") +_RECORDER_EXECUTE_OPTION = typer.Option(False, "--execute", help="Execute the recorded plan.") +_RECORDER_ACK_OPTION = typer.Option( + False, + "--acknowledge-paid", + help="Acknowledge that configured commands may incur charges.", +) +_RECORDER_COST_CAP_OPTION = typer.Option( + None, + "--cost-cap-usd", + help="Explicit cap that must cover the configured maximum estimate.", +) +_FINALIZE_EVIDENCE_OPTION = typer.Option( + ..., + "--evidence", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Normalized recorder evidence JSON.", +) +_FINALIZE_REVIEWS_OPTION = typer.Option( + ..., + "--reviews", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Independent adjudication reviews JSON.", +) +_FINALIZE_OUTPUT_OPTION = typer.Option(None, "--output", help="Final Phase 40 results path.") +_REHEARSAL_OUTPUT_OPTION = typer.Option(..., "--output-dir", help="New rehearsal bundle directory.") +_REHEARSAL_REVISION_OPTION = typer.Option("HEAD", "--revision", help="Git revision to pin.") +_PREFLIGHT_VERSIONS_OPTION = typer.Option( + ..., + "--runtime-versions", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Declared agent CLI versions JSON; no version command is executed.", +) +_PREFLIGHT_OUTPUT_OPTION = typer.Option(..., "--output", help="New preflight JSON path.") +_PREFLIGHT_INPUT_OPTION = typer.Option( + ..., + "--preflight", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Campaign preflight JSON.", +) +_OPTIONAL_PREFLIGHT_INPUT = typer.Option( + None, + "--preflight", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional campaign preflight binding.", +) +_OPTIONAL_REVIEW_POLICY_INPUT = typer.Option( + None, + "--review-policy", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional operator/reviewer separation policy declaration.", +) +_OPTIONAL_REVIEWER_TRUST_INPUT = typer.Option( + None, + "--reviewer-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional reviewer Ed25519 public-key trust declaration.", +) +_OPTIONAL_ATTESTATIONS_INPUT = typer.Option( + None, + "--attestations", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional signed reviewer attestation bundle.", +) +_OPTIONAL_REVIEWER_AUTHORITY_INPUT = typer.Option( + None, + "--reviewer-authority", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional organization benchmark authority declaration.", +) +_OPTIONAL_REVIEWER_ENROLLMENTS_INPUT = typer.Option( + None, + "--reviewer-enrollments", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional authority-signed reviewer enrollment bundle.", +) +_OPTIONAL_CAMPAIGN_ENVELOPE_INPUT = typer.Option( + None, + "--campaign-envelope", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional authority-signed finalized campaign envelope.", +) +_OPTIONAL_AUTHORITY_ROOT_LEDGER_INPUT = typer.Option( + None, + "--authority-root-ledger", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional signed authority-root ledger.", +) +_OPTIONAL_TRANSPARENCY_PROOF_INPUT = typer.Option( + None, + "--transparency-proof", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional campaign-envelope transparency inclusion proof.", +) +_OPTIONAL_TRANSPARENCY_CONSISTENCY_INPUT = typer.Option( + None, + "--transparency-consistency-proof", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional compact Merkle consistency proof.", +) +_OPTIONAL_TRANSPARENCY_WITNESS_TRUST_INPUT = typer.Option( + None, + "--transparency-witness-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional transparency witness trust declaration.", +) +_OPTIONAL_WITNESS_CHECKPOINT_INPUT = typer.Option( + None, + "--witness-checkpoint", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional signed witness checkpoint bundle.", +) +_STATUS_EVIDENCE_OPTION = typer.Option( + None, + "--evidence", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional normalized evidence JSON.", +) +_STATUS_REVIEWS_OPTION = typer.Option( + None, + "--reviews", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional pending template or completed reviews JSON.", +) +_STATUS_RESULTS_OPTION = typer.Option( + None, + "--results", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Optional finalized results JSON.", +) +_AUTHORITY_ROTATION_GENERATION_OPTION = typer.Option(..., "--generation", min=2) +_AUTHORITY_PREDECESSOR_OPTION = typer.Option( + ..., + "--predecessor", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_AUTHORITY_SUCCESSOR_OPTION = typer.Option( + ..., + "--successor", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_AUTHORITY_GENERATIONS_OPTION = typer.Option( + ..., + "--generations", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="JSON object containing generations and optional revocations.", +) +_TRANSPARENCY_LOG_ID_OPTION = typer.Option(..., "--log-id") +_TRANSPARENCY_ENTRIES_OPTION = typer.Option( + ..., + "--entries", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="JSON array of transparency log entries.", +) +_TRANSPARENCY_LOG_INPUT = typer.Option( + ..., + "--log", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_AUTHORITY_ROOT_LEDGER_INPUT = typer.Option( + ..., + "--authority-root-ledger", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_TRANSPARENCY_TREE_HEAD_INPUT = typer.Option( + ..., + "--tree-head", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_TRANSPARENCY_LEAF_INDEX_OPTION = typer.Option(..., "--leaf-index", min=0) +_TRANSPARENCY_CURRENT_LOG_INPUT = typer.Option( + ..., + "--current-log", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_TRANSPARENCY_PREVIOUS_TREE_HEAD_INPUT = typer.Option( + ..., + "--previous-tree-head", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_TRANSPARENCY_CURRENT_TREE_HEAD_INPUT = typer.Option( + ..., + "--current-tree-head", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_WITNESS_TRUST_DECLARATION_INPUT = typer.Option( + ..., + "--declaration", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_WITNESS_TRUST_INPUT = typer.Option( + ..., + "--witness-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_TRANSPARENCY_CONSISTENCY_INPUT = typer.Option( + ..., + "--consistency-proof", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_REGISTRY_OPTION = typer.Option( + ..., + "--registry", + help="Checkpoint registry JSONL path.", +) +_CHECKPOINT_REGISTRY_ID_OPTION = typer.Option(..., "--registry-id") +_CHECKPOINT_PEER_TRUST_INPUT = typer.Option( + ..., + "--peer-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_SOURCE_PEER_OPTION = typer.Option(..., "--source-peer-id") +_CHECKPOINT_SEQUENCE_OPTION = typer.Option( + None, + "--sequence", + min=0, + help="Registry sequence to export; defaults to the latest record.", +) +_WITNESS_CHECKPOINT_INPUT = typer.Option( + ..., + "--witness-checkpoint", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_PACKET_INPUT = typer.Option( + ..., + "--packet", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_RANGE_BUNDLE_INPUT = typer.Option( + ..., + "--range-bundle", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_START_SEQUENCE_OPTION = typer.Option( + 0, + "--start-sequence", + min=0, +) +_CHECKPOINT_MAX_RECORDS_OPTION = typer.Option( + 100, + "--max-records", + min=1, + max=1000, +) +_CHECKPOINT_ACKNOWLEDGING_PEER_OPTION = typer.Option( + ..., + "--acknowledging-peer-id", +) +_CHECKPOINT_ACKNOWLEDGEMENT_INPUT = typer.Option( + ..., + "--acknowledgement", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_CURSOR_LEDGER_OPTION = typer.Option( + ..., + "--cursor-ledger", + help="Peer acknowledgement cursor JSONL path.", +) +_CHECKPOINT_PREDECESSOR_PEER_TRUST_INPUT = typer.Option( + ..., + "--predecessor-peer-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_SUCCESSOR_PEER_TRUST_INPUT = typer.Option( + ..., + "--successor-peer-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_PEER_TRUST_GENERATIONS_INPUT = typer.Option( + ..., + "--generations", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT = typer.Option( + None, + "--peer-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT = typer.Option( + None, + "--peer-trust-ledger", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_PEER_TRUST_LEDGER_INPUT = typer.Option( + ..., + "--peer-trust-ledger", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, +) +_CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT = typer.Option( + ..., + "--descriptor", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Mode 0600 loopback gossip descriptor.", +) +_CHECKPOINT_GOSSIP_DESCRIPTOR_OUTPUT = typer.Option( + ..., + "--descriptor", + help="New mode 0600 loopback gossip descriptor path.", +) +_CHECKPOINT_GOSSIP_BUNDLES_INPUT = typer.Option( + ..., + "--range-bundles", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="JSON object containing pre-signed range bundles.", +) +_CHECKPOINT_GOSSIP_MAX_REQUESTS_OPTION = typer.Option( + 64, + "--max-requests", + min=1, + max=1024, + help="Authenticated requests accepted before deterministic shutdown.", +) +_CHECKPOINT_GOSSIP_LIFETIME_OPTION = typer.Option( + 300.0, + "--lifetime-seconds", + min=1.0, + max=3600.0, + help="Maximum listener lifetime without deterministic shutdown.", +) +_CHECKPOINT_TLS_CERTIFICATE_INPUT = typer.Option( + ..., + "--certificate", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Active TLS leaf certificate PEM.", +) +_CHECKPOINT_TLS_PRIVATE_KEY_INPUT = typer.Option( + ..., + "--private-key", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="TLS private key with owner-only permissions.", +) +_CHECKPOINT_TLS_CA_INPUT = typer.Option( + ..., + "--certificate-authority", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="CA bundle used for mutual certificate verification.", +) +_CHECKPOINT_TLS_TRUST_INPUT = typer.Option( + ..., + "--tls-trust", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Peer-signed TLS trust JSON.", +) +_CHECKPOINT_TLS_ENROLLMENTS_INPUT = typer.Option( + ..., + "--enrollments", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="JSON object containing signed TLS enrollments.", +) +_CHECKPOINT_TLS_TEMPLATE_INPUT = typer.Option( + ..., + "--template", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Private-key-free TLS enrollment template JSON.", +) +_CHECKPOINT_TLS_REVOCATION_TEMPLATE_INPUT = typer.Option( + ..., "--revocation-template", exists=True, file_okay=True, dir_okay=False, readable=True +) +_CHECKPOINT_TLS_PEER_TRUST_INPUT = typer.Option( + ..., "--peer-trust", exists=True, file_okay=True, dir_okay=False, readable=True +) +_CHECKPOINT_TLS_KEY_ID_OPTION = typer.Option( + ..., + "--key-id", + help="Eligible peer identity key ID used for the detached signature.", +) +_CHECKPOINT_TLS_SIGNATURE_OPTION = typer.Option( + ..., + "--signature-base64", + help="Detached Ed25519 signature over the template signing payload.", +) +_CHECKPOINT_TLS_PREDECESSOR_INPUT = typer.Option( + None, + "--predecessor", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Required predecessor enrollment for generation greater than one.", +) +_CHECKPOINT_TLS_BIND_HOST_OPTION = typer.Option( + ..., + "--bind-host", + help="Explicit bind IP.", +) +_CHECKPOINT_TLS_ADVERTISED_HOST_OPTION = typer.Option( + ..., + "--advertised-host", + help="Explicit client-visible IP.", +) +_CHECKPOINT_TLS_ALLOWED_CLIENTS_OPTION = typer.Option( + ..., + "--allow-client-address", + help="Allowed client IP; repeat for each address.", +) +_CHECKPOINT_TLS_CLIENT_PEER_OPTION = typer.Option( + ..., + "--client-peer-id", + help="Enrolled client peer identity.", +) +_CHECKPOINT_TLS_SERVER_HOSTNAME_OPTION = typer.Option( + ..., + "--server-hostname", + help="DNS name that must match the server certificate SAN.", +) +_CHECKPOINT_TLS_ALLOWED_SERVERS_OPTION = typer.Option( + ..., + "--allow-server-address", + help="Allowed server IP; repeat for each address.", +) +_CHECKPOINT_GOSSIP_SYNC_AUDIT_OPTION = typer.Option( + ..., + "--sync-audit", + help="Durable checkpoint gossip sync audit JSONL path.", +) +_CHECKPOINT_GOSSIP_SYNC_ROUNDS_OPTION = typer.Option( + 16, + "--max-rounds", + min=1, + max=100, +) +_CHECKPOINT_GOSSIP_SYNC_ATTEMPTS_OPTION = typer.Option( + 3, + "--max-attempts", + min=1, + max=10, + help="Attempts per status or range request.", +) + + +def _write_new_evidence(path: Path, payload: str) -> None: + """Publish evidence atomically without replacing an existing path.""" + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as temporary: + temporary.write(payload) + temporary.flush() + os.fsync(temporary.fileno()) + temporary_path = Path(temporary.name) + os.link(temporary_path, path) + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) def _get_container() -> Any: @@ -126,3 +767,2699 @@ def run_dedup() -> None: for s in result.scores: style = "green" if s.dedup_rate >= 0.5 else "yellow" console.print(f" {s.scenario:<25} [{style}]{s.dedup_rate:.0%}[/{style}]") + + +@benchmark_app.command("agent-cli") +def compare_agent_clis( + manifest_path: Path = _AGENT_CLI_MANIFEST_OPTION, + results_path: Path = _AGENT_CLI_RESULTS_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Compare recorded Codex, Claude Code, and Morphic-controlled trials.""" + from benchmarks.agent_cli_comparison import ( + AgentCliManifest, + RecordedResults, + evaluate_recorded_results, + ) + + try: + manifest = AgentCliManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + results = RecordedResults.model_validate_json(results_path.read_text(encoding="utf-8")) + report = evaluate_recorded_results(manifest, results) + except (OSError, ValueError) as exc: + console.print(f"[red]Invalid agent CLI benchmark input: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + console.print(report.to_json(), markup=False) + return + + from rich.table import Table + + table = Table(title=f"Agent CLI Same-task Benchmark: {report.benchmark_id}") + table.add_column("Arm", style="cyan") + table.add_column("Complete", justify="right") + table.add_column("Accepted", justify="right") + table.add_column("Verified", justify="right") + table.add_column("Median sec", justify="right") + table.add_column("Mean USD", justify="right") + table.add_column("Interventions", justify="right") + table.add_column("Recovery", justify="right") + table.add_column("Handoff", justify="right") + for arm, metrics in report.arms.items(): + recovery = "n/a" if metrics.recovery_rate is None else f"{metrics.recovery_rate:.0%}" + table.add_row( + arm, + f"{metrics.completion_rate:.0%}", + f"{metrics.accepted_patch_rate:.0%}", + f"{metrics.verification_rate:.0%}", + f"{metrics.median_elapsed_seconds:.1f}", + f"{metrics.mean_cost_usd:.4f}", + f"{metrics.mean_human_interventions:.2f}", + recovery, + f"{metrics.context_handoff_score:.0%}", + ) + console.print(table) + console.print("[bold]Metric leaders[/bold]") + for metric, leaders in report.leaders.items(): + names = ", ".join(leaders) if leaders else "n/a" + console.print(f" {metric}: {names}") + console.print("[dim]No composite score; metric leaders are reported independently.[/dim]") + + +@benchmark_app.command("agent-cli-record") +def record_agent_cli_trials( + manifest_path: Path = _AGENT_CLI_MANIFEST_OPTION, + config_path: Path = _RECORDER_CONFIG_OPTION, + worktree_root: Path | None = _RECORDER_WORKTREE_OPTION, + source_root: Path | None = _RECORDER_SOURCE_OPTION, + evidence_path: Path | None = _RECORDER_EVIDENCE_OPTION, + execute: bool = _RECORDER_EXECUTE_OPTION, + acknowledge_paid: bool = _RECORDER_ACK_OPTION, + cost_cap_usd: float | None = _RECORDER_COST_CAP_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Plan or explicitly execute isolated same-task CLI trials.""" + from benchmarks.agent_cli_comparison import AgentCliManifest + from benchmarks.agent_cli_recorder import ( + AgentCliRecorderConfig, + AgentCliTrialRecorder, + GitWorktreeManager, + LocalCommandRunner, + build_recording_plan, + validate_execution_consent, + ) + + try: + manifest = AgentCliManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + config = AgentCliRecorderConfig.model_validate_json(config_path.read_text(encoding="utf-8")) + plan = build_recording_plan(manifest, config) + except (OSError, ValueError) as exc: + console.print(f"[red]Invalid agent CLI recorder input: {exc}[/red]") + raise typer.Exit(code=1) from None + + if not execute: + if as_json: + typer.echo(plan.to_json()) + else: + console.print( + f"Recorder plan trials={plan.trial_count} " + f"estimated_max=${plan.estimated_max_cost_usd:.6f} execute=false" + ) + return + + try: + validate_execution_consent( + plan, + acknowledged_paid=acknowledge_paid, + cost_cap_usd=cost_cap_usd, + ) + if worktree_root is None: + raise ValueError("--worktree-root is required with --execute") + if evidence_path is None: + raise ValueError("--evidence is required with --execute") + if not evidence_path.parent.exists(): + raise ValueError("evidence output parent must already exist") + if evidence_path.exists(): + raise ValueError("evidence output already exists") + except ValueError as exc: + console.print(f"[red]Recorder execution refused: {exc}[/red]") + raise typer.Exit(code=2) from None + + recorder = AgentCliTrialRecorder( + worktree_manager=GitWorktreeManager(), + command_runner=LocalCommandRunner(), + ) + try: + evidence = _run( + recorder.record( + manifest=manifest, + config=config, + source_root=(source_root or Path.cwd()).resolve(), + worktree_root=worktree_root.resolve(), + acknowledged_paid=acknowledge_paid, + cost_cap_usd=cost_cap_usd, + ) + ) + _write_new_evidence(evidence_path, evidence.to_json()) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Recorder execution failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(evidence.to_json()) + else: + console.print(f"Recorded {len(evidence.trials)} isolated trials to {evidence_path}") + + +@benchmark_app.command("agent-cli-finalize") +def finalize_agent_cli_trials( + manifest_path: Path = _AGENT_CLI_MANIFEST_OPTION, + evidence_path: Path = _FINALIZE_EVIDENCE_OPTION, + reviews_path: Path = _FINALIZE_REVIEWS_OPTION, + preflight_path: Path | None = _OPTIONAL_PREFLIGHT_INPUT, + review_policy_path: Path | None = _OPTIONAL_REVIEW_POLICY_INPUT, + reviewer_trust_path: Path | None = _OPTIONAL_REVIEWER_TRUST_INPUT, + attestations_path: Path | None = _OPTIONAL_ATTESTATIONS_INPUT, + reviewer_authority_path: Path | None = _OPTIONAL_REVIEWER_AUTHORITY_INPUT, + reviewer_enrollments_path: Path | None = _OPTIONAL_REVIEWER_ENROLLMENTS_INPUT, + authority_root_ledger_path: Path | None = _OPTIONAL_AUTHORITY_ROOT_LEDGER_INPUT, + output_path: Path | None = _FINALIZE_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Join normalized receipts and reviews into Phase 40 observations.""" + from benchmarks.agent_cli_adjudication import ( + AdjudicationReviews, + RecordedEvidence, + finalize_recorded_results, + finalized_results_json, + ) + from benchmarks.agent_cli_comparison import AgentCliManifest + + try: + manifest = AgentCliManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + evidence = RecordedEvidence.model_validate_json(evidence_path.read_text(encoding="utf-8")) + reviews = AdjudicationReviews.model_validate_json(reviews_path.read_text(encoding="utf-8")) + review_policy = None + reviewer_trust = None + attestations = None + reviewer_authority = None + reviewer_enrollments = None + authority_root_ledger = None + if reviews.preflight_sha256 is not None and preflight_path is None: + raise ValueError("--preflight is required for a preflight-bound review") + if preflight_path is not None: + from benchmarks.agent_cli_preflight import ( + CampaignPreflight, + validate_review_bindings, + ) + + preflight = CampaignPreflight.model_validate_json( + preflight_path.read_text(encoding="utf-8") + ) + validate_review_bindings(preflight, evidence, reviews) + if reviews.review_policy_sha256 is not None and review_policy_path is None: + raise ValueError("--review-policy is required for a policy-bound review") + if review_policy_path is not None: + from benchmarks.agent_cli_review_policy import ( + ReviewerPolicyDeclaration, + build_reviewer_policy, + validate_reviewer_separation, + ) + + declaration = ReviewerPolicyDeclaration.model_validate_json( + review_policy_path.read_text(encoding="utf-8") + ) + review_policy = build_reviewer_policy(declaration) + validate_reviewer_separation(review_policy, reviews) + if reviews.reviewer_trust_sha256 is not None and reviewer_trust_path is None: + raise ValueError("--reviewer-trust is required for a trust-bound review") + if reviews.reviewer_trust_sha256 is not None and attestations_path is None: + raise ValueError("--attestations is required for a trust-bound review") + if reviewer_trust_path is not None: + if review_policy is None: + raise ValueError("--review-policy is required with --reviewer-trust") + from benchmarks.agent_cli_attestation import ( + ReviewerTrustDeclaration, + build_reviewer_trust, + ) + + trust_declaration = ReviewerTrustDeclaration.model_validate_json( + reviewer_trust_path.read_text(encoding="utf-8") + ) + reviewer_trust = build_reviewer_trust(trust_declaration, review_policy) + if ( + reviewer_trust is not None + and reviewer_trust.reviewer_authority_sha256 is not None + and reviewer_authority_path is None + ): + raise ValueError("--reviewer-authority is required for authority-bound trust") + if ( + reviewer_trust is not None + and reviewer_trust.reviewer_authority_sha256 is not None + and reviewer_enrollments_path is None + ): + raise ValueError("--reviewer-enrollments is required for authority-bound trust") + if attestations_path is not None: + from benchmarks.agent_cli_attestation import ReviewAttestationBundle + + attestations = ReviewAttestationBundle.model_validate_json( + attestations_path.read_text(encoding="utf-8") + ) + if reviewer_authority_path is not None: + from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + build_benchmark_authority, + ) + + reviewer_authority = build_benchmark_authority( + BenchmarkAuthorityDeclaration.model_validate_json( + reviewer_authority_path.read_text(encoding="utf-8") + ) + ) + if reviewer_enrollments_path is not None: + from benchmarks.agent_cli_authority import ReviewerEnrollmentBundle + + reviewer_enrollments = ReviewerEnrollmentBundle.model_validate_json( + reviewer_enrollments_path.read_text(encoding="utf-8") + ) + if authority_root_ledger_path is not None: + from benchmarks.agent_cli_transparency import SignedAuthorityRootLedger + + authority_root_ledger = SignedAuthorityRootLedger.model_validate_json( + authority_root_ledger_path.read_text(encoding="utf-8") + ) + results = finalize_recorded_results( + manifest, + evidence, + reviews, + review_policy=review_policy, + reviewer_trust=reviewer_trust, + attestations=attestations, + reviewer_authority=reviewer_authority, + reviewer_enrollments=reviewer_enrollments, + authority_root_ledger=authority_root_ledger, + ) + payload = finalized_results_json(results) + if output_path is not None: + if not output_path.parent.exists(): + raise ValueError("output parent must already exist") + if output_path.exists(): + raise ValueError("output already exists") + _write_new_evidence(output_path, payload) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI adjudication failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(payload) + else: + console.print(f"Finalized {len(results.observations)} benchmark observations") + + +@benchmark_app.command("agent-cli-preflight") +def preflight_agent_cli_campaign( + manifest_path: Path = _AGENT_CLI_MANIFEST_OPTION, + config_path: Path = _RECORDER_CONFIG_OPTION, + runtime_versions_path: Path = _PREFLIGHT_VERSIONS_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + source_root: Path | None = _RECORDER_SOURCE_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Validate one pinned campaign without launching an agent or authorizing execution.""" + from benchmarks.agent_cli_comparison import AgentCliManifest + from benchmarks.agent_cli_preflight import ( + RuntimeVersionBundle, + build_campaign_preflight, + resolve_git_revision, + ) + from benchmarks.agent_cli_recorder import AgentCliRecorderConfig + + try: + manifest = AgentCliManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + config = AgentCliRecorderConfig.model_validate_json(config_path.read_text(encoding="utf-8")) + versions = RuntimeVersionBundle.model_validate_json( + runtime_versions_path.read_text(encoding="utf-8") + ) + if not output_path.parent.exists(): + raise ValueError("preflight output parent must already exist") + if output_path.exists(): + raise ValueError("preflight output already exists") + source = (source_root or Path.cwd()).resolve() + resolved = _run(resolve_git_revision(source, manifest.task.workspace_revision)) + report = build_campaign_preflight( + manifest, + config, + versions, + resolved_revision=resolved, + ) + _write_new_evidence(output_path, report.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI preflight failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(report.to_json()) + else: + console.print( + f"Preflight ready trials={report.trial_count} " + f"estimated_max=${report.estimated_max_cost_usd:.6f} " + "execution_authorized=false" + ) + + +@benchmark_app.command("agent-cli-review-template") +def create_agent_cli_review_template( + preflight_path: Path = _PREFLIGHT_INPUT_OPTION, + evidence_path: Path = _FINALIZE_EVIDENCE_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + review_policy_path: Path | None = _OPTIONAL_REVIEW_POLICY_INPUT, + reviewer_trust_path: Path | None = _OPTIONAL_REVIEWER_TRUST_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Generate null independent review decisions bound to exact campaign evidence.""" + from benchmarks.agent_cli_adjudication import RecordedEvidence + from benchmarks.agent_cli_preflight import CampaignPreflight, build_review_template + + try: + preflight = CampaignPreflight.model_validate_json( + preflight_path.read_text(encoding="utf-8") + ) + evidence = RecordedEvidence.model_validate_json(evidence_path.read_text(encoding="utf-8")) + review_policy_sha256 = None + reviewer_trust_sha256 = None + review_policy = None + if review_policy_path is not None: + from benchmarks.agent_cli_review_policy import ( + ReviewerPolicyDeclaration, + build_reviewer_policy, + validate_reviewer_policy_capacity, + ) + + declaration = ReviewerPolicyDeclaration.model_validate_json( + review_policy_path.read_text(encoding="utf-8") + ) + review_policy = build_reviewer_policy(declaration) + if review_policy.benchmark_id != preflight.benchmark_id: + raise ValueError("review policy benchmark_id does not match preflight") + validate_reviewer_policy_capacity( + review_policy, + decision_count=len(evidence.trials), + ) + review_policy_sha256 = review_policy.policy_sha256 + if reviewer_trust_path is not None: + if review_policy is None: + raise ValueError("--review-policy is required with --reviewer-trust") + from benchmarks.agent_cli_attestation import ( + ReviewerTrustDeclaration, + build_reviewer_trust, + ) + + trust_declaration = ReviewerTrustDeclaration.model_validate_json( + reviewer_trust_path.read_text(encoding="utf-8") + ) + reviewer_trust = build_reviewer_trust(trust_declaration, review_policy) + reviewer_trust_sha256 = reviewer_trust.reviewer_trust_sha256 + if not output_path.parent.exists(): + raise ValueError("review template output parent must already exist") + if output_path.exists(): + raise ValueError("review template output already exists") + template = build_review_template( + preflight, + evidence, + review_policy_sha256=review_policy_sha256, + reviewer_trust_sha256=reviewer_trust_sha256, + ) + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI review template failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(template.to_json()) + else: + console.print(f"Created {len(template.decisions)} bound review decisions") + + +@benchmark_app.command("agent-cli-attestation-template") +def create_agent_cli_attestation_template( + reviews_path: Path = _FINALIZE_REVIEWS_OPTION, + review_policy_path: Path = _OPTIONAL_REVIEW_POLICY_INPUT, + reviewer_trust_path: Path = _OPTIONAL_REVIEWER_TRUST_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create canonical reviewer signing payloads without reading private keys.""" + from benchmarks.agent_cli_adjudication import AdjudicationReviews + from benchmarks.agent_cli_attestation import ( + ReviewerTrustDeclaration, + build_review_attestation_template, + build_reviewer_trust, + ) + from benchmarks.agent_cli_review_policy import ( + ReviewerPolicyDeclaration, + build_reviewer_policy, + ) + + try: + if review_policy_path is None or reviewer_trust_path is None: + raise ValueError("--review-policy and --reviewer-trust are required") + reviews = AdjudicationReviews.model_validate_json(reviews_path.read_text(encoding="utf-8")) + policy = build_reviewer_policy( + ReviewerPolicyDeclaration.model_validate_json( + review_policy_path.read_text(encoding="utf-8") + ) + ) + trust = build_reviewer_trust( + ReviewerTrustDeclaration.model_validate_json( + reviewer_trust_path.read_text(encoding="utf-8") + ), + policy, + ) + template = build_review_attestation_template(policy, trust, reviews) + if not output_path.parent.exists(): + raise ValueError("attestation template output parent must already exist") + if output_path.exists(): + raise ValueError("attestation template output already exists") + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI attestation template failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(template.to_json()) + else: + console.print(f"Created {len(template.requests)} reviewer signing requests") + + +@benchmark_app.command("agent-cli-reviewer-enrollment-template") +def create_agent_cli_reviewer_enrollment_template( + review_policy_path: Path = _OPTIONAL_REVIEW_POLICY_INPUT, + reviewer_trust_path: Path = _OPTIONAL_REVIEWER_TRUST_INPUT, + reviewer_authority_path: Path = _OPTIONAL_REVIEWER_AUTHORITY_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create authority signing payloads for reviewer keys without private keys.""" + from benchmarks.agent_cli_attestation import ( + ReviewerTrustDeclaration, + build_reviewer_trust, + ) + from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + build_benchmark_authority, + build_reviewer_enrollment_template, + ) + from benchmarks.agent_cli_review_policy import ( + ReviewerPolicyDeclaration, + build_reviewer_policy, + ) + + try: + if ( + review_policy_path is None + or reviewer_trust_path is None + or reviewer_authority_path is None + ): + raise ValueError( + "--review-policy, --reviewer-trust, and --reviewer-authority are required" + ) + policy = build_reviewer_policy( + ReviewerPolicyDeclaration.model_validate_json( + review_policy_path.read_text(encoding="utf-8") + ) + ) + trust = build_reviewer_trust( + ReviewerTrustDeclaration.model_validate_json( + reviewer_trust_path.read_text(encoding="utf-8") + ), + policy, + ) + authority = build_benchmark_authority( + BenchmarkAuthorityDeclaration.model_validate_json( + reviewer_authority_path.read_text(encoding="utf-8") + ) + ) + template = build_reviewer_enrollment_template(authority, policy, trust) + if not output_path.parent.exists(): + raise ValueError("reviewer enrollment output parent must already exist") + if output_path.exists(): + raise ValueError("reviewer enrollment output already exists") + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI reviewer enrollment template failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(template.to_json()) + else: + console.print(f"Created {len(template.requests)} authority enrollment requests") + + +@benchmark_app.command("agent-cli-authority-rotation-template") +def create_agent_cli_authority_rotation_template( + generation: int = _AUTHORITY_ROTATION_GENERATION_OPTION, + predecessor_path: Path = _AUTHORITY_PREDECESSOR_OPTION, + successor_path: Path = _AUTHORITY_SUCCESSOR_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create a predecessor-root signing payload without reading private keys.""" + from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + build_benchmark_authority, + ) + from benchmarks.agent_cli_transparency import build_authority_rotation_request + + try: + predecessor = build_benchmark_authority( + BenchmarkAuthorityDeclaration.model_validate_json( + predecessor_path.read_text(encoding="utf-8") + ) + ) + successor = build_benchmark_authority( + BenchmarkAuthorityDeclaration.model_validate_json( + successor_path.read_text(encoding="utf-8") + ) + ) + request = build_authority_rotation_request( + generation=generation, + predecessor=predecessor, + successor=successor, + ) + if not output_path.parent.exists(): + raise ValueError("authority rotation output parent must already exist") + if output_path.exists(): + raise ValueError("authority rotation output already exists") + _write_new_evidence(output_path, request.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI authority rotation template failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(request.to_json()) + else: + console.print(f"Created authority rotation request for generation {generation}") + + +@benchmark_app.command("agent-cli-authority-root-ledger-template") +def create_agent_cli_authority_root_ledger_template( + generations_path: Path = _AUTHORITY_GENERATIONS_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Verify a rotation chain and create the active-root ledger signing payload.""" + from benchmarks.agent_cli_transparency import ( + AuthorityRootGeneration, + build_authority_root_ledger_request, + ) + + try: + payload = json.loads(generations_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("authority root generations JSON must be an object") + generations_payload = payload.get("generations") + if not isinstance(generations_payload, list): + raise ValueError("authority root generations must be a JSON array") + revocations = payload.get("revoked_authority_sha256", []) + if not isinstance(revocations, list) or not all( + isinstance(value, str) for value in revocations + ): + raise ValueError("authority root revocations must be a string array") + request = build_authority_root_ledger_request( + tuple(AuthorityRootGeneration.model_validate(item) for item in generations_payload), + revoked_authority_sha256=tuple(revocations), + ) + if not output_path.parent.exists(): + raise ValueError("authority root ledger output parent must already exist") + if output_path.exists(): + raise ValueError("authority root ledger output already exists") + _write_new_evidence(output_path, request.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI authority root ledger failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(request.to_json()) + else: + console.print("Created active-root ledger signing request") + + +@benchmark_app.command("agent-cli-transparency-log") +def create_agent_cli_transparency_log( + log_id: str = _TRANSPARENCY_LOG_ID_OPTION, + entries_path: Path = _TRANSPARENCY_ENTRIES_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Build a complete deterministic Merkle transparency log offline.""" + from benchmarks.agent_cli_transparency import ( + TransparencyLogEntry, + build_transparency_log, + ) + + try: + payload = json.loads(entries_path.read_text(encoding="utf-8")) + if not isinstance(payload, list): + raise ValueError("transparency entries JSON must be an array") + log = build_transparency_log( + log_id, + tuple(TransparencyLogEntry.model_validate(item) for item in payload), + ) + if not output_path.parent.exists(): + raise ValueError("transparency log output parent must already exist") + if output_path.exists(): + raise ValueError("transparency log output already exists") + _write_new_evidence(output_path, log.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI transparency log failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(log.to_json()) + else: + console.print(f"Created transparency log tree_size={log.tree_size}") + + +@benchmark_app.command("agent-cli-transparency-tree-head-template") +def create_agent_cli_transparency_tree_head_template( + log_path: Path = _TRANSPARENCY_LOG_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create an active-root signing payload for one Merkle tree head.""" + from benchmarks.agent_cli_transparency import ( + SignedAuthorityRootLedger, + TransparencyLog, + build_transparency_tree_head_request, + ) + + try: + log = TransparencyLog.model_validate_json(log_path.read_text(encoding="utf-8")) + ledger = SignedAuthorityRootLedger.model_validate_json( + authority_root_ledger_path.read_text(encoding="utf-8") + ) + request = build_transparency_tree_head_request(log, ledger) + if not output_path.parent.exists(): + raise ValueError("transparency tree head output parent must already exist") + if output_path.exists(): + raise ValueError("transparency tree head output already exists") + _write_new_evidence(output_path, request.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI transparency tree head failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(request.to_json()) + else: + console.print("Created transparency tree-head signing request") + + +@benchmark_app.command("agent-cli-transparency-proof") +def create_agent_cli_transparency_proof( + log_path: Path = _TRANSPARENCY_LOG_INPUT, + tree_head_path: Path = _TRANSPARENCY_TREE_HEAD_INPUT, + leaf_index: int = _TRANSPARENCY_LEAF_INDEX_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Build an inclusion proof for one entry under a signed tree head.""" + from benchmarks.agent_cli_transparency import ( + SignedTransparencyTreeHead, + TransparencyLog, + build_transparency_inclusion_proof, + ) + + try: + log = TransparencyLog.model_validate_json(log_path.read_text(encoding="utf-8")) + tree_head = SignedTransparencyTreeHead.model_validate_json( + tree_head_path.read_text(encoding="utf-8") + ) + proof = build_transparency_inclusion_proof( + log, + leaf_index=leaf_index, + tree_head=tree_head, + ) + if not output_path.parent.exists(): + raise ValueError("transparency proof output parent must already exist") + if output_path.exists(): + raise ValueError("transparency proof output already exists") + _write_new_evidence(output_path, proof.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI transparency proof failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(proof.to_json()) + else: + console.print(f"Created transparency proof for leaf {leaf_index}") + + +@benchmark_app.command("agent-cli-transparency-consistency-proof") +def create_agent_cli_transparency_consistency_proof( + current_log_path: Path = _TRANSPARENCY_CURRENT_LOG_INPUT, + previous_tree_head_path: Path = _TRANSPARENCY_PREVIOUS_TREE_HEAD_INPUT, + current_tree_head_path: Path = _TRANSPARENCY_CURRENT_TREE_HEAD_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Build a compact append-only proof between two signed tree heads.""" + from benchmarks.agent_cli_transparency import ( + SignedAuthorityRootLedger, + SignedTransparencyTreeHead, + TransparencyLog, + build_transparency_consistency_proof, + ) + + try: + current_log = TransparencyLog.model_validate_json( + current_log_path.read_text(encoding="utf-8") + ) + previous_tree_head = SignedTransparencyTreeHead.model_validate_json( + previous_tree_head_path.read_text(encoding="utf-8") + ) + current_tree_head = SignedTransparencyTreeHead.model_validate_json( + current_tree_head_path.read_text(encoding="utf-8") + ) + ledger = SignedAuthorityRootLedger.model_validate_json( + authority_root_ledger_path.read_text(encoding="utf-8") + ) + proof = build_transparency_consistency_proof( + current_log, + previous_tree_head=previous_tree_head, + current_tree_head=current_tree_head, + authority_root_ledger=ledger, + ) + if not output_path.parent.exists(): + raise ValueError("consistency proof output parent must already exist") + if output_path.exists(): + raise ValueError("consistency proof output already exists") + _write_new_evidence(output_path, proof.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI consistency proof failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(proof.to_json()) + else: + console.print( + "Created compact consistency proof " + f"{proof.previous_tree_head.statement.tree_size}->" + f"{proof.current_tree_head.statement.tree_size}" + ) + + +@benchmark_app.command("agent-cli-witness-trust") +def create_agent_cli_witness_trust( + declaration_path: Path = _WITNESS_TRUST_DECLARATION_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Normalize an intersecting transparency-witness quorum declaration.""" + from benchmarks.agent_cli_witness import ( + TransparencyWitnessTrustDeclaration, + build_transparency_witness_trust, + ) + + try: + trust = build_transparency_witness_trust( + TransparencyWitnessTrustDeclaration.model_validate_json( + declaration_path.read_text(encoding="utf-8") + ) + ) + if not output_path.parent.exists(): + raise ValueError("witness trust output parent must already exist") + if output_path.exists(): + raise ValueError("witness trust output already exists") + _write_new_evidence(output_path, trust.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI witness trust failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(trust.to_json()) + else: + console.print( + f"Created witness trust minimum_distinct_witnesses={trust.minimum_distinct_witnesses}" + ) + + +@benchmark_app.command("agent-cli-witness-checkpoint-template") +def create_agent_cli_witness_checkpoint_template( + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + consistency_proof_path: Path = _TRANSPARENCY_CONSISTENCY_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create private-key-free signing requests for a witness checkpoint.""" + from benchmarks.agent_cli_transparency import ( + SignedAuthorityRootLedger, + TransparencyConsistencyProof, + ) + from benchmarks.agent_cli_witness import ( + TransparencyWitnessTrust, + build_witness_checkpoint_template, + ) + + try: + trust = TransparencyWitnessTrust.model_validate_json( + witness_trust_path.read_text(encoding="utf-8") + ) + proof = TransparencyConsistencyProof.model_validate_json( + consistency_proof_path.read_text(encoding="utf-8") + ) + ledger = SignedAuthorityRootLedger.model_validate_json( + authority_root_ledger_path.read_text(encoding="utf-8") + ) + template = build_witness_checkpoint_template(trust, proof, ledger) + if not output_path.parent.exists(): + raise ValueError("witness checkpoint output parent must already exist") + if output_path.exists(): + raise ValueError("witness checkpoint output already exists") + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI witness checkpoint failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(template.to_json()) + else: + console.print(f"Created {len(template.requests)} witness signing requests") + + +@benchmark_app.command("agent-cli-checkpoint-peer-trust") +def create_agent_cli_checkpoint_peer_trust( + declaration_path: Path = _WITNESS_TRUST_DECLARATION_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Normalize public keys trusted to exchange checkpoint records.""" + from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerTrustDeclaration, + build_checkpoint_peer_trust, + ) + + try: + trust = build_checkpoint_peer_trust( + CheckpointPeerTrustDeclaration.model_validate_json( + declaration_path.read_text(encoding="utf-8") + ) + ) + if not output_path.parent.exists(): + raise ValueError("checkpoint peer trust output parent must already exist") + if output_path.exists(): + raise ValueError("checkpoint peer trust output already exists") + _write_new_evidence(output_path, trust.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint peer trust failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(trust.to_json()) + else: + console.print(f"Created checkpoint peer trust with {len(trust.keys)} keys") + + +@benchmark_app.command("agent-cli-checkpoint-peer-tls-enrollment-template") +def create_agent_cli_checkpoint_peer_tls_enrollment_template( + certificate_path: Path = _CHECKPOINT_TLS_CERTIFICATE_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + peer_id: str = typer.Option(..., "--peer-id", help="Peer identity to enroll."), + generation: int = typer.Option( + ..., + "--generation", + min=1, + help="Monotonic TLS enrollment generation.", + ), + predecessor_path: Path | None = _CHECKPOINT_TLS_PREDECESSOR_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create a private-key-free peer-signed TLS enrollment request.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsEnrollment, + build_checkpoint_peer_tls_enrollment_template, + ) + + try: + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + predecessor = ( + CheckpointPeerTlsEnrollment.model_validate_json( + predecessor_path.read_text(encoding="utf-8") + ) + if predecessor_path is not None + else None + ) + template = build_checkpoint_peer_tls_enrollment_template( + certificate_path.read_bytes(), + peer_trust, + peer_id=peer_id, + generation=generation, + predecessor=predecessor, + ) + if not output_path.parent.exists(): + raise ValueError("TLS enrollment output parent must already exist") + if output_path.exists(): + raise ValueError("TLS enrollment output already exists") + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI TLS enrollment template failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(template.to_json()) + else: + console.print( + f"Created TLS enrollment signing request peer={peer_id} generation={generation}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-peer-tls-enrollment") +def create_agent_cli_checkpoint_peer_tls_enrollment( + template_path: Path = _CHECKPOINT_TLS_TEMPLATE_INPUT, + certificate_path: Path = _CHECKPOINT_TLS_CERTIFICATE_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + key_id: str = _CHECKPOINT_TLS_KEY_ID_OPTION, + signature_base64: str = _CHECKPOINT_TLS_SIGNATURE_OPTION, + predecessor_path: Path | None = _CHECKPOINT_TLS_PREDECESSOR_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Verify an external identity signature and finalize one TLS enrollment.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsEnrollment, + CheckpointPeerTlsEnrollmentTemplate, + build_signed_checkpoint_peer_tls_enrollment, + ) + + try: + template = CheckpointPeerTlsEnrollmentTemplate.model_validate_json( + template_path.read_text(encoding="utf-8") + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + predecessor = ( + CheckpointPeerTlsEnrollment.model_validate_json( + predecessor_path.read_text(encoding="utf-8") + ) + if predecessor_path is not None + else None + ) + enrollment = build_signed_checkpoint_peer_tls_enrollment( + template, + certificate_path.read_bytes(), + peer_trust, + key_id=key_id, + signature_base64=signature_base64, + predecessor=predecessor, + ) + if not output_path.parent.exists(): + raise ValueError("TLS enrollment output parent must already exist") + if output_path.exists(): + raise ValueError("TLS enrollment output already exists") + _write_new_evidence(output_path, enrollment.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI TLS enrollment failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(enrollment.to_json()) + else: + console.print( + "Verified TLS enrollment " + f"peer={enrollment.statement.peer_id} " + f"generation={enrollment.statement.generation}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-peer-tls-revocation-template") +def create_agent_cli_checkpoint_peer_tls_revocation_template( + tls_trust_path: Path = _CHECKPOINT_TLS_TRUST_INPUT, + peer_trust_path: Path = _CHECKPOINT_TLS_PEER_TRUST_INPUT, + peer_id: str = typer.Option(..., "--peer-id"), + generation: int = typer.Option(..., "--generation", min=1), + reason: str = typer.Option(..., "--reason"), + revoked_at: str = typer.Option(..., "--revoked-at"), + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create a private-key-free signed TLS revocation request.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsTrust, + build_checkpoint_peer_tls_revocation_template, + ) + + try: + tls_trust = CheckpointPeerTlsTrust.model_validate_json( + tls_trust_path.read_text(encoding="utf-8") + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + template = build_checkpoint_peer_tls_revocation_template( + tls_trust, + peer_trust, + peer_id=peer_id, + generation=generation, + reason=reason, + revoked_at=revoked_at, + ) + if output_path.exists(): + raise ValueError("TLS revocation template output already exists") + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI TLS revocation template failed: {exc}[/red]") + raise typer.Exit(code=1) from None + typer.echo(template.to_json()) if as_json else console.print("Created TLS revocation template") + + +@benchmark_app.command("agent-cli-checkpoint-peer-tls-revocation") +def create_agent_cli_checkpoint_peer_tls_revocation( + template_path: Path = _CHECKPOINT_TLS_REVOCATION_TEMPLATE_INPUT, + peer_trust_path: Path = _CHECKPOINT_TLS_PEER_TRUST_INPUT, + key_id: str = _CHECKPOINT_TLS_KEY_ID_OPTION, + signature_base64: str = _CHECKPOINT_TLS_SIGNATURE_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Verify a peer signature and finalize a TLS revocation artifact.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsRevocationTemplate, + build_signed_checkpoint_peer_tls_revocation, + ) + + try: + template = CheckpointPeerTlsRevocationTemplate.model_validate_json( + template_path.read_text(encoding="utf-8") + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + revocation = build_signed_checkpoint_peer_tls_revocation( + template, peer_trust, key_id=key_id, signature_base64=signature_base64 + ) + if output_path.exists(): + raise ValueError("TLS revocation output already exists") + _write_new_evidence(output_path, revocation.model_dump_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI TLS revocation failed: {exc}[/red]") + raise typer.Exit(code=1) from None + typer.echo(revocation.model_dump_json()) if as_json else console.print( + "Verified TLS revocation" + ) + + +@benchmark_app.command("agent-cli-checkpoint-peer-tls-trust") +def create_agent_cli_checkpoint_peer_tls_trust( + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + enrollments_path: Path = _CHECKPOINT_TLS_ENROLLMENTS_INPUT, + revocations_path: Path | None = _CHECKPOINT_TLS_REVOCATIONS_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Verify enrollment chains and publish active per-peer TLS pins.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsEnrollment, + CheckpointPeerTlsRevocation, + build_checkpoint_peer_tls_trust, + ) + + try: + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + payload = json.loads(enrollments_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or not isinstance(payload.get("enrollments"), list): + raise ValueError("TLS enrollments JSON must contain an enrollments array") + revocations: tuple[CheckpointPeerTlsRevocation, ...] = () + if revocations_path is not None: + revocation_payload = json.loads(revocations_path.read_text(encoding="utf-8")) + if not isinstance(revocation_payload, dict) or not isinstance( + revocation_payload.get("revocations"), list + ): + raise ValueError("TLS revocations JSON must contain a revocations array") + revocations = tuple( + CheckpointPeerTlsRevocation.model_validate(item) + for item in revocation_payload["revocations"] + ) + trust = build_checkpoint_peer_tls_trust( + peer_trust, + tuple( + CheckpointPeerTlsEnrollment.model_validate(item) for item in payload["enrollments"] + ), + revocations, + ) + if not output_path.parent.exists(): + raise ValueError("TLS trust output parent must already exist") + if output_path.exists(): + raise ValueError("TLS trust output already exists") + _write_new_evidence(output_path, trust.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI TLS trust failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(trust.to_json()) + else: + console.print( + "Created checkpoint TLS trust " + f"peers={len(trust.peer_ids())} enrollments={len(trust.enrollments)}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-peer-trust-rotation-template") +def create_agent_cli_checkpoint_peer_trust_rotation_template( + predecessor_path: Path = _CHECKPOINT_PREDECESSOR_PEER_TRUST_INPUT, + successor_path: Path = _CHECKPOINT_SUCCESSOR_PEER_TRUST_INPUT, + generation: int = _AUTHORITY_ROTATION_GENERATION_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create strict-majority predecessor-peer rollover signing requests.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_peer_trust_ledger import ( + build_checkpoint_peer_trust_rotation_template, + ) + + try: + predecessor = CheckpointPeerTrust.model_validate_json( + predecessor_path.read_text(encoding="utf-8") + ) + successor = CheckpointPeerTrust.model_validate_json( + successor_path.read_text(encoding="utf-8") + ) + template = build_checkpoint_peer_trust_rotation_template( + predecessor, + successor, + generation=generation, + ) + if not output_path.parent.exists(): + raise ValueError("peer trust rotation output parent must already exist") + if output_path.exists(): + raise ValueError("peer trust rotation output already exists") + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI peer trust rotation failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(template.to_json()) + else: + console.print( + "Created peer trust rotation requests " + f"generation={generation} quorum=" + f"{template.minimum_distinct_peer_signatures}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-peer-trust-ledger") +def create_agent_cli_checkpoint_peer_trust_ledger( + generations_path: Path = _CHECKPOINT_PEER_TRUST_GENERATIONS_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Verify and publish a complete peer-trust generation ledger.""" + from benchmarks.agent_cli_peer_trust_ledger import ( + CheckpointPeerTrustGeneration, + build_checkpoint_peer_trust_ledger, + ) + + try: + payload = json.loads(generations_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("peer trust generations JSON must be an object") + generations_payload = payload.get("generations") + if not isinstance(generations_payload, list): + raise ValueError("peer trust generations must be a JSON array") + ledger = build_checkpoint_peer_trust_ledger( + tuple( + CheckpointPeerTrustGeneration.model_validate(item) for item in generations_payload + ) + ) + if not output_path.parent.exists(): + raise ValueError("peer trust ledger output parent must already exist") + if output_path.exists(): + raise ValueError("peer trust ledger output already exists") + _write_new_evidence(output_path, ledger.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI peer trust ledger failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(ledger.to_json()) + else: + console.print(f"Created peer trust ledger active_generation={ledger.active_generation}") + + +def _load_checkpoint_registry_dependencies( + witness_trust_path: Path, + authority_root_ledger_path: Path, +) -> tuple[Any, Any]: + from benchmarks.agent_cli_transparency import SignedAuthorityRootLedger + from benchmarks.agent_cli_witness import TransparencyWitnessTrust + + witness_trust = TransparencyWitnessTrust.model_validate_json( + witness_trust_path.read_text(encoding="utf-8") + ) + ledger = SignedAuthorityRootLedger.model_validate_json( + authority_root_ledger_path.read_text(encoding="utf-8") + ) + return witness_trust, ledger + + +@benchmark_app.command("agent-cli-checkpoint-registry-status") +def show_agent_cli_checkpoint_registry_status( + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Replay and verify a local registry without changing it.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointRegistryStore + + try: + witness_trust, ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + snapshot = CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ).replay(witness_trust, ledger) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint registry status failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(snapshot.to_json()) + else: + console.print( + "Verified checkpoint registry " + f"records={snapshot.record_count} tree_size={snapshot.current_tree_size}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-registry-store") +def store_agent_cli_checkpoint_registry_record( + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + consistency_proof_path: Path = _TRANSPARENCY_CONSISTENCY_INPUT, + witness_checkpoint_path: Path = _WITNESS_CHECKPOINT_INPUT, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Verify and durably append one witnessed checkpoint.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointRegistryStore + from benchmarks.agent_cli_transparency import TransparencyConsistencyProof + from benchmarks.agent_cli_witness import SignedWitnessCheckpoint + + try: + if not registry_path.parent.exists(): + raise ValueError("checkpoint registry parent must already exist") + witness_trust, ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + proof = TransparencyConsistencyProof.model_validate_json( + consistency_proof_path.read_text(encoding="utf-8") + ) + checkpoint = SignedWitnessCheckpoint.model_validate_json( + witness_checkpoint_path.read_text(encoding="utf-8") + ) + record = CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ).append(proof, checkpoint, witness_trust, ledger) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint registry store failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(record.to_json()) + else: + console.print( + f"Stored checkpoint registry sequence={record.sequence} " + f"tree_size={record.checkpoint.statement.current_tree_size}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-registry-export-template") +def export_agent_cli_checkpoint_registry_template( + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + source_peer_id: str = _CHECKPOINT_SOURCE_PEER_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + sequence: int | None = _CHECKPOINT_SEQUENCE_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create a detached peer-signing request for one exact registry record.""" + from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerTrust, + CheckpointRegistryStore, + build_checkpoint_exchange_request, + ) + + try: + witness_trust, ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + snapshot = CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ).replay(witness_trust, ledger) + if not snapshot.records: + raise ValueError("checkpoint registry has no records to export") + selected_sequence = len(snapshot.records) - 1 if sequence is None else sequence + if selected_sequence >= len(snapshot.records): + raise ValueError("checkpoint registry sequence does not exist") + request = build_checkpoint_exchange_request( + snapshot.records[selected_sequence], + peer_trust, + source_peer_id=source_peer_id, + ) + if not output_path.parent.exists(): + raise ValueError("checkpoint export output parent must already exist") + if output_path.exists(): + raise ValueError("checkpoint export output already exists") + _write_new_evidence(output_path, request.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint registry export failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(request.to_json()) + else: + console.print( + "Created checkpoint exchange signing request " + f"sequence={request.statement.record_sequence}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-registry-import") +def import_agent_cli_checkpoint_registry_packet( + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + packet_path: Path = _CHECKPOINT_PACKET_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Authenticate and atomically import one peer checkpoint packet.""" + from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerTrust, + CheckpointRegistryStore, + SignedCheckpointExchangePacket, + ) + + try: + if not registry_path.parent.exists(): + raise ValueError("checkpoint registry parent must already exist") + witness_trust, ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + packet = SignedCheckpointExchangePacket.model_validate_json( + packet_path.read_text(encoding="utf-8") + ) + record = CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ).import_packet(packet, peer_trust, witness_trust, ledger) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint registry import failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(record.to_json()) + else: + console.print( + f"Imported checkpoint registry sequence={record.sequence} " + f"tree_size={record.checkpoint.statement.current_tree_size}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-range-export-template") +def export_agent_cli_checkpoint_range_template( + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + source_peer_id: str = _CHECKPOINT_SOURCE_PEER_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + start_sequence: int = _CHECKPOINT_START_SEQUENCE_OPTION, + max_records: int = _CHECKPOINT_MAX_RECORDS_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create one signing request for a bounded contiguous registry range.""" + from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerTrust, + CheckpointRegistryStore, + build_checkpoint_range_request, + ) + + try: + witness_trust, ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + snapshot = CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ).replay(witness_trust, ledger) + if start_sequence >= snapshot.record_count: + raise ValueError("checkpoint range start sequence does not exist") + records = snapshot.records[start_sequence : start_sequence + max_records] + request = build_checkpoint_range_request( + records, + peer_trust, + source_peer_id=source_peer_id, + ) + if not output_path.parent.exists(): + raise ValueError("checkpoint range output parent must already exist") + if output_path.exists(): + raise ValueError("checkpoint range output already exists") + _write_new_evidence(output_path, request.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint range export failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(request.to_json()) + else: + console.print( + "Created checkpoint range signing request " + f"{request.statement.first_sequence}->" + f"{request.statement.last_sequence}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-range-import") +def import_agent_cli_checkpoint_range( + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + range_bundle_path: Path = _CHECKPOINT_RANGE_BUNDLE_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Authenticate a range and atomically append its missing suffix.""" + from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerTrust, + CheckpointRegistryStore, + SignedCheckpointRangeBundle, + ) + + try: + if not registry_path.parent.exists(): + raise ValueError("checkpoint registry parent must already exist") + witness_trust, ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + bundle = SignedCheckpointRangeBundle.model_validate_json( + range_bundle_path.read_text(encoding="utf-8") + ) + snapshot = CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ).import_range_bundle(bundle, peer_trust, witness_trust, ledger) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint range import failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(snapshot.to_json()) + else: + console.print( + "Imported checkpoint range " + f"records={snapshot.record_count} tree_size={snapshot.current_tree_size}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-acknowledgement-template") +def create_agent_cli_checkpoint_acknowledgement_template( + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + range_bundle_path: Path = _CHECKPOINT_RANGE_BUNDLE_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + acknowledging_peer_id: str = _CHECKPOINT_ACKNOWLEDGING_PEER_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create a receiver signing request for an exactly applied range head.""" + from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerTrust, + CheckpointRegistryStore, + SignedCheckpointRangeBundle, + build_checkpoint_acknowledgement_request, + ) + + try: + witness_trust, ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + bundle = SignedCheckpointRangeBundle.model_validate_json( + range_bundle_path.read_text(encoding="utf-8") + ) + snapshot = CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ).replay(witness_trust, ledger) + request = build_checkpoint_acknowledgement_request( + bundle, + snapshot, + peer_trust, + acknowledging_peer_id=acknowledging_peer_id, + ) + if not output_path.parent.exists(): + raise ValueError("checkpoint acknowledgement output parent must already exist") + if output_path.exists(): + raise ValueError("checkpoint acknowledgement output already exists") + _write_new_evidence(output_path, request.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint acknowledgement failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(request.to_json()) + else: + console.print( + "Created checkpoint acknowledgement signing request " + f"sequence={request.statement.acknowledged_record_sequence}" + ) + + +def _load_checkpoint_peer_trust_source( + peer_trust_path: Path | None, + peer_trust_ledger_path: Path | None, +) -> Any: + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_peer_trust_ledger import CheckpointPeerTrustLedger + + if (peer_trust_path is None) == (peer_trust_ledger_path is None): + raise ValueError("exactly one of --peer-trust or --peer-trust-ledger is required") + if peer_trust_path is not None: + return CheckpointPeerTrust.model_validate_json(peer_trust_path.read_text(encoding="utf-8")) + assert peer_trust_ledger_path is not None + return CheckpointPeerTrustLedger.model_validate_json( + peer_trust_ledger_path.read_text(encoding="utf-8") + ) + + +def _active_checkpoint_peer_trust(peer_trust_source: Any) -> Any: + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_peer_trust_ledger import CheckpointPeerTrustLedger + + if isinstance(peer_trust_source, CheckpointPeerTrust): + return peer_trust_source + if isinstance(peer_trust_source, CheckpointPeerTrustLedger): + return peer_trust_source.active_trust + raise ValueError("unsupported checkpoint peer trust source") + + +def _build_checkpoint_mtls_client( + *, + descriptor_path: Path, + peer_trust_source: Any, + tls_trust_path: Path, + client_peer_id: str, + certificate_path: Path, + private_key_path: Path, + certificate_authority_path: Path, + server_hostname: str, + allowed_server_addresses: list[str], +) -> Any: + from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsTrust, + verify_checkpoint_peer_tls_trust, + ) + from benchmarks.agent_cli_gossip_tls_transport import ( + CheckpointMutualTlsGossipClient, + ) + + active_peer_trust = _active_checkpoint_peer_trust(peer_trust_source) + tls_trust = CheckpointPeerTlsTrust.model_validate_json( + tls_trust_path.read_text(encoding="utf-8") + ) + verify_checkpoint_peer_tls_trust(tls_trust, active_peer_trust) + return CheckpointMutualTlsGossipClient( + descriptor_path=descriptor_path, + client_peer_id=client_peer_id, + tls_trust=tls_trust, + certificate_path=certificate_path, + private_key_path=private_key_path, + certificate_authority_path=certificate_authority_path, + server_hostname=server_hostname, + allowed_server_addresses=frozenset(allowed_server_addresses), + ) + + +@benchmark_app.command("agent-cli-checkpoint-cursor-store") +def store_agent_cli_checkpoint_cursor( + cursor_ledger_path: Path = _CHECKPOINT_CURSOR_LEDGER_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + acknowledgement_path: Path = _CHECKPOINT_ACKNOWLEDGEMENT_INPUT, + peer_trust_path: Path | None = _OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT, + peer_trust_ledger_path: Path | None = (_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT), + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Verify and durably store one monotonic peer acknowledgement.""" + from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerCursorStore, + SignedCheckpointAcknowledgement, + ) + + try: + if not cursor_ledger_path.parent.exists(): + raise ValueError("checkpoint cursor ledger parent must already exist") + peer_trust = _load_checkpoint_peer_trust_source( + peer_trust_path, + peer_trust_ledger_path, + ) + acknowledgement = SignedCheckpointAcknowledgement.model_validate_json( + acknowledgement_path.read_text(encoding="utf-8") + ) + record = CheckpointPeerCursorStore( + cursor_ledger_path, + registry_id=registry_id, + ).append(acknowledgement, peer_trust) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint cursor store failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(record.to_json()) + else: + console.print( + "Stored checkpoint peer cursor " + f"sequence={record.acknowledgement.statement.acknowledged_record_sequence}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-cursor-status") +def show_agent_cli_checkpoint_cursor_status( + cursor_ledger_path: Path = _CHECKPOINT_CURSOR_LEDGER_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + peer_trust_path: Path | None = _OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT, + peer_trust_ledger_path: Path | None = (_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT), + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Replay peer acknowledgements without changing the cursor ledger.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerCursorStore + + try: + peer_trust = _load_checkpoint_peer_trust_source( + peer_trust_path, + peer_trust_ledger_path, + ) + snapshot = CheckpointPeerCursorStore( + cursor_ledger_path, + registry_id=registry_id, + ).replay(peer_trust) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint cursor status failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(snapshot.to_json()) + else: + console.print( + "Verified checkpoint peer cursors " + f"records={snapshot.cursor_count} peers={len(snapshot.positions)}" + ) + + +def _load_checkpoint_gossip_bundles(path: Path) -> tuple[Any, ...]: + from benchmarks.agent_cli_checkpoint_registry import SignedCheckpointRangeBundle + + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or not isinstance(payload.get("bundles"), list): + raise ValueError("checkpoint gossip bundles must contain a JSON bundles array") + return tuple(SignedCheckpointRangeBundle.model_validate(item) for item in payload["bundles"]) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-serve") +def serve_agent_cli_checkpoint_gossip( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_OUTPUT, + range_bundles_path: Path = _CHECKPOINT_GOSSIP_BUNDLES_INPUT, + cursor_ledger_path: Path = _CHECKPOINT_CURSOR_LEDGER_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + source_peer_id: str = _CHECKPOINT_SOURCE_PEER_OPTION, + peer_trust_path: Path | None = _OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT, + peer_trust_ledger_path: Path | None = (_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT), + max_requests: int = _CHECKPOINT_GOSSIP_MAX_REQUESTS_OPTION, + lifetime_seconds: float = _CHECKPOINT_GOSSIP_LIFETIME_OPTION, +) -> None: + """Explicitly serve pre-signed checkpoint artifacts on loopback only.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerCursorStore + from benchmarks.agent_cli_gossip import CheckpointGossipService + from benchmarks.agent_cli_gossip_transport import CheckpointGossipServer + + try: + if not descriptor_path.parent.exists(): + raise ValueError("checkpoint gossip descriptor parent must already exist") + if not cursor_ledger_path.parent.exists(): + raise ValueError("checkpoint gossip cursor parent must already exist") + peer_trust = _load_checkpoint_peer_trust_source( + peer_trust_path, + peer_trust_ledger_path, + ) + service = CheckpointGossipService( + registry_id=registry_id, + source_peer_id=source_peer_id, + range_bundles=_load_checkpoint_gossip_bundles(range_bundles_path), + cursor_store=CheckpointPeerCursorStore( + cursor_ledger_path, + registry_id=registry_id, + ), + peer_trust=peer_trust, + ) + + async def run_server() -> None: + server = CheckpointGossipServer( + descriptor_path=descriptor_path, + registry_id=registry_id, + source_peer_id=source_peer_id, + handler=service, + max_requests=max_requests, + ) + async with server: + console.print( + "Checkpoint gossip listening on loopback " + f"descriptor={descriptor_path} max_requests={max_requests}" + ) + await server.serve_until_stopped( + lifetime_timeout_seconds=lifetime_seconds, + ) + + _run(run_server()) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint gossip server failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + +@benchmark_app.command("agent-cli-checkpoint-gossip-status") +def show_agent_cli_checkpoint_gossip_status( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Query authenticated range availability without changing either peer.""" + from benchmarks.agent_cli_gossip import fetch_checkpoint_gossip_status + + try: + status = _run(fetch_checkpoint_gossip_status(descriptor_path=descriptor_path)) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint gossip status failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo(json.dumps(status, ensure_ascii=False, sort_keys=True)) + else: + console.print( + "Authenticated checkpoint gossip " + f"source={status['source_peer_id']} " + f"ranges={len(status['available_ranges'])}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-mtls-serve") +def serve_agent_cli_checkpoint_gossip_mtls( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_OUTPUT, + range_bundles_path: Path = _CHECKPOINT_GOSSIP_BUNDLES_INPUT, + cursor_ledger_path: Path = _CHECKPOINT_CURSOR_LEDGER_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + source_peer_id: str = _CHECKPOINT_SOURCE_PEER_OPTION, + peer_trust_path: Path | None = _OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT, + peer_trust_ledger_path: Path | None = (_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT), + tls_trust_path: Path = _CHECKPOINT_TLS_TRUST_INPUT, + certificate_path: Path = _CHECKPOINT_TLS_CERTIFICATE_INPUT, + private_key_path: Path = _CHECKPOINT_TLS_PRIVATE_KEY_INPUT, + certificate_authority_path: Path = _CHECKPOINT_TLS_CA_INPUT, + bind_host: str = _CHECKPOINT_TLS_BIND_HOST_OPTION, + advertised_host: str = _CHECKPOINT_TLS_ADVERTISED_HOST_OPTION, + allowed_client_addresses: list[str] = _CHECKPOINT_TLS_ALLOWED_CLIENTS_OPTION, + max_requests: int = _CHECKPOINT_GOSSIP_MAX_REQUESTS_OPTION, + lifetime_seconds: float = _CHECKPOINT_GOSSIP_LIFETIME_OPTION, +) -> None: + """Serve checkpoint gossip over TLS 1.3 mutual authentication.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerCursorStore + from benchmarks.agent_cli_gossip import CheckpointGossipService + from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsTrust, + verify_checkpoint_peer_tls_trust, + ) + from benchmarks.agent_cli_gossip_tls_transport import ( + CheckpointMutualTlsGossipServer, + ) + + try: + if not descriptor_path.parent.exists(): + raise ValueError("checkpoint gossip descriptor parent must already exist") + if not cursor_ledger_path.parent.exists(): + raise ValueError("checkpoint gossip cursor parent must already exist") + peer_trust = _load_checkpoint_peer_trust_source( + peer_trust_path, + peer_trust_ledger_path, + ) + tls_trust = CheckpointPeerTlsTrust.model_validate_json( + tls_trust_path.read_text(encoding="utf-8") + ) + verify_checkpoint_peer_tls_trust( + tls_trust, + _active_checkpoint_peer_trust(peer_trust), + ) + service = CheckpointGossipService( + registry_id=registry_id, + source_peer_id=source_peer_id, + range_bundles=_load_checkpoint_gossip_bundles(range_bundles_path), + cursor_store=CheckpointPeerCursorStore( + cursor_ledger_path, + registry_id=registry_id, + ), + peer_trust=peer_trust, + ) + + async def run_server() -> None: + server = CheckpointMutualTlsGossipServer( + descriptor_path=descriptor_path, + bind_host=bind_host, + advertised_host=advertised_host, + registry_id=registry_id, + server_peer_id=source_peer_id, + handler=service, + tls_trust=tls_trust, + certificate_path=certificate_path, + private_key_path=private_key_path, + certificate_authority_path=certificate_authority_path, + allowed_client_addresses=frozenset(allowed_client_addresses), + max_requests=max_requests, + ) + async with server: + console.print( + "Checkpoint gossip listening with TLS 1.3 mutual auth " + f"descriptor={descriptor_path} max_requests={max_requests}" + ) + await server.serve_until_stopped(lifetime_timeout_seconds=lifetime_seconds) + + _run(run_server()) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint mTLS server failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + +@benchmark_app.command("agent-cli-checkpoint-gossip-mtls-status") +def show_agent_cli_checkpoint_gossip_mtls_status( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + peer_trust_path: Path = _CHECKPOINT_PEER_TRUST_INPUT, + tls_trust_path: Path = _CHECKPOINT_TLS_TRUST_INPUT, + client_peer_id: str = _CHECKPOINT_TLS_CLIENT_PEER_OPTION, + certificate_path: Path = _CHECKPOINT_TLS_CERTIFICATE_INPUT, + private_key_path: Path = _CHECKPOINT_TLS_PRIVATE_KEY_INPUT, + certificate_authority_path: Path = _CHECKPOINT_TLS_CA_INPUT, + server_hostname: str = _CHECKPOINT_TLS_SERVER_HOSTNAME_OPTION, + allowed_server_addresses: list[str] = _CHECKPOINT_TLS_ALLOWED_SERVERS_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Query checkpoint status over pinned TLS 1.3 mutual authentication.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointPeerTrust + from benchmarks.agent_cli_gossip import fetch_checkpoint_gossip_status + + try: + peer_trust = CheckpointPeerTrust.model_validate_json( + peer_trust_path.read_text(encoding="utf-8") + ) + client = _build_checkpoint_mtls_client( + descriptor_path=descriptor_path, + peer_trust_source=peer_trust, + tls_trust_path=tls_trust_path, + client_peer_id=client_peer_id, + certificate_path=certificate_path, + private_key_path=private_key_path, + certificate_authority_path=certificate_authority_path, + server_hostname=server_hostname, + allowed_server_addresses=allowed_server_addresses, + ) + status = _run( + fetch_checkpoint_gossip_status( + descriptor_path=descriptor_path, + request_sender=client, + ) + ) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint mTLS status failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo( + json.dumps( + {**status, "tls_expiry_warnings": client.expiry_warnings}, + ensure_ascii=False, + sort_keys=True, + ) + ) + else: + console.print( + "Mutually authenticated checkpoint gossip " + f"source={status.get('source_peer_id', 'unknown')} " + f"ranges={len(status.get('available_ranges', []))} " + f"tls_expiry_warnings={len(client.expiry_warnings)}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-mtls-fetch") +def fetch_agent_cli_checkpoint_gossip_mtls_range( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + start_sequence: int = _CHECKPOINT_START_SEQUENCE_OPTION, + max_records: int = _CHECKPOINT_MAX_RECORDS_OPTION, + peer_trust_path: Path | None = _OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT, + peer_trust_ledger_path: Path | None = (_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT), + tls_trust_path: Path = _CHECKPOINT_TLS_TRUST_INPUT, + client_peer_id: str = _CHECKPOINT_TLS_CLIENT_PEER_OPTION, + certificate_path: Path = _CHECKPOINT_TLS_CERTIFICATE_INPUT, + private_key_path: Path = _CHECKPOINT_TLS_PRIVATE_KEY_INPUT, + certificate_authority_path: Path = _CHECKPOINT_TLS_CA_INPUT, + server_hostname: str = _CHECKPOINT_TLS_SERVER_HOSTNAME_OPTION, + allowed_server_addresses: list[str] = _CHECKPOINT_TLS_ALLOWED_SERVERS_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Fetch and verify one signed range over pinned mutual TLS.""" + from benchmarks.agent_cli_gossip import fetch_signed_checkpoint_range + + try: + if not output_path.parent.exists(): + raise ValueError("checkpoint gossip output parent must already exist") + if output_path.exists(): + raise ValueError("checkpoint gossip output already exists") + peer_trust = _load_checkpoint_peer_trust_source( + peer_trust_path, + peer_trust_ledger_path, + ) + client = _build_checkpoint_mtls_client( + descriptor_path=descriptor_path, + peer_trust_source=peer_trust, + tls_trust_path=tls_trust_path, + client_peer_id=client_peer_id, + certificate_path=certificate_path, + private_key_path=private_key_path, + certificate_authority_path=certificate_authority_path, + server_hostname=server_hostname, + allowed_server_addresses=allowed_server_addresses, + ) + bundle = _run( + fetch_signed_checkpoint_range( + descriptor_path=descriptor_path, + start_sequence=start_sequence, + max_records=max_records, + peer_trust=peer_trust, + request_sender=client, + ) + ) + _write_new_evidence(output_path, bundle.to_json()) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint mTLS fetch failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo(bundle.to_json()) + else: + console.print( + "Fetched mutually authenticated checkpoint range " + f"{bundle.statement.first_sequence}->{bundle.statement.last_sequence}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-mtls-ack") +def submit_agent_cli_checkpoint_gossip_mtls_acknowledgement( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + acknowledgement_path: Path = _CHECKPOINT_ACKNOWLEDGEMENT_INPUT, + peer_trust_path: Path | None = _OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT, + peer_trust_ledger_path: Path | None = (_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT), + tls_trust_path: Path = _CHECKPOINT_TLS_TRUST_INPUT, + client_peer_id: str = _CHECKPOINT_TLS_CLIENT_PEER_OPTION, + certificate_path: Path = _CHECKPOINT_TLS_CERTIFICATE_INPUT, + private_key_path: Path = _CHECKPOINT_TLS_PRIVATE_KEY_INPUT, + certificate_authority_path: Path = _CHECKPOINT_TLS_CA_INPUT, + server_hostname: str = _CHECKPOINT_TLS_SERVER_HOSTNAME_OPTION, + allowed_server_addresses: list[str] = _CHECKPOINT_TLS_ALLOWED_SERVERS_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Submit a signed acknowledgement over pinned mutual TLS.""" + from benchmarks.agent_cli_checkpoint_registry import SignedCheckpointAcknowledgement + from benchmarks.agent_cli_gossip import submit_signed_checkpoint_acknowledgement + + try: + peer_trust = _load_checkpoint_peer_trust_source( + peer_trust_path, + peer_trust_ledger_path, + ) + client = _build_checkpoint_mtls_client( + descriptor_path=descriptor_path, + peer_trust_source=peer_trust, + tls_trust_path=tls_trust_path, + client_peer_id=client_peer_id, + certificate_path=certificate_path, + private_key_path=private_key_path, + certificate_authority_path=certificate_authority_path, + server_hostname=server_hostname, + allowed_server_addresses=allowed_server_addresses, + ) + acknowledgement = SignedCheckpointAcknowledgement.model_validate_json( + acknowledgement_path.read_text(encoding="utf-8") + ) + record = _run( + submit_signed_checkpoint_acknowledgement( + descriptor_path=descriptor_path, + acknowledgement=acknowledgement, + request_sender=client, + ) + ) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint mTLS ack failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo(record.to_json()) + else: + console.print( + "Submitted mutually authenticated checkpoint acknowledgement " + f"sequence={record.acknowledgement.statement.acknowledged_record_sequence}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-fetch") +def fetch_agent_cli_checkpoint_gossip_range( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + start_sequence: int = _CHECKPOINT_START_SEQUENCE_OPTION, + max_records: int = _CHECKPOINT_MAX_RECORDS_OPTION, + peer_trust_path: Path | None = _OPTIONAL_CHECKPOINT_PEER_TRUST_INPUT, + peer_trust_ledger_path: Path | None = (_OPTIONAL_CHECKPOINT_PEER_TRUST_LEDGER_INPUT), + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Fetch and independently verify one already signed checkpoint range.""" + from benchmarks.agent_cli_gossip import fetch_signed_checkpoint_range + + try: + if not output_path.parent.exists(): + raise ValueError("checkpoint gossip output parent must already exist") + if output_path.exists(): + raise ValueError("checkpoint gossip output already exists") + peer_trust = _load_checkpoint_peer_trust_source( + peer_trust_path, + peer_trust_ledger_path, + ) + bundle = _run( + fetch_signed_checkpoint_range( + descriptor_path=descriptor_path, + start_sequence=start_sequence, + max_records=max_records, + peer_trust=peer_trust, + ) + ) + _write_new_evidence(output_path, bundle.to_json()) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint gossip fetch failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo(bundle.to_json()) + else: + console.print( + "Fetched authenticated checkpoint range " + f"{bundle.statement.first_sequence}->{bundle.statement.last_sequence}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-ack") +def submit_agent_cli_checkpoint_gossip_acknowledgement( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + acknowledgement_path: Path = _CHECKPOINT_ACKNOWLEDGEMENT_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Submit a signed range acknowledgement to its source peer.""" + from benchmarks.agent_cli_checkpoint_registry import SignedCheckpointAcknowledgement + from benchmarks.agent_cli_gossip import submit_signed_checkpoint_acknowledgement + + try: + acknowledgement = SignedCheckpointAcknowledgement.model_validate_json( + acknowledgement_path.read_text(encoding="utf-8") + ) + record = _run( + submit_signed_checkpoint_acknowledgement( + descriptor_path=descriptor_path, + acknowledgement=acknowledgement, + ) + ) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint gossip ack failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo(record.to_json()) + else: + console.print( + "Submitted authenticated checkpoint acknowledgement " + f"sequence={record.acknowledgement.statement.acknowledged_record_sequence}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-sync") +def sync_agent_cli_checkpoint_gossip( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + sync_audit_path: Path = _CHECKPOINT_GOSSIP_SYNC_AUDIT_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + source_peer_id: str = _CHECKPOINT_SOURCE_PEER_OPTION, + peer_trust_ledger_path: Path = _CHECKPOINT_PEER_TRUST_LEDGER_INPUT, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + max_rounds: int = _CHECKPOINT_GOSSIP_SYNC_ROUNDS_OPTION, + max_records: int = _CHECKPOINT_MAX_RECORDS_OPTION, + max_attempts: int = _CHECKPOINT_GOSSIP_SYNC_ATTEMPTS_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Run one bounded resumable pull loop without reading private keys.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointRegistryStore + from benchmarks.agent_cli_gossip_sync import ( + CheckpointGossipSyncAuditStore, + CheckpointGossipSyncPolicy, + run_checkpoint_gossip_sync, + ) + from benchmarks.agent_cli_peer_trust_ledger import CheckpointPeerTrustLedger + + try: + if not registry_path.parent.exists(): + raise ValueError("checkpoint gossip registry parent must already exist") + if not sync_audit_path.parent.exists(): + raise ValueError("checkpoint gossip sync audit parent must already exist") + witness_trust, authority_ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + peer_trust_ledger = CheckpointPeerTrustLedger.model_validate_json( + peer_trust_ledger_path.read_text(encoding="utf-8") + ) + retry_delays = tuple(min(0.05 * (2**index), 1.0) for index in range(max_attempts - 1)) + result = _run( + run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ), + audit_store=CheckpointGossipSyncAuditStore( + sync_audit_path, + registry_id=registry_id, + source_peer_id=source_peer_id, + ), + peer_trust_ledger=peer_trust_ledger, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=CheckpointGossipSyncPolicy( + max_rounds=max_rounds, + max_records=max_records, + max_attempts_per_request=max_attempts, + retry_delays_seconds=retry_delays, + ), + ) + ) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint gossip sync failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo(result.to_json()) + else: + console.print( + "Checkpoint gossip sync stopped " + f"reason={result.stop_reason} records={result.local_record_count} " + f"imported={result.records_imported} retries={result.retries}" + ) + + +@benchmark_app.command("agent-cli-checkpoint-gossip-mtls-sync") +def sync_agent_cli_checkpoint_gossip_mtls( + descriptor_path: Path = _CHECKPOINT_GOSSIP_DESCRIPTOR_INPUT, + registry_path: Path = _CHECKPOINT_REGISTRY_OPTION, + sync_audit_path: Path = _CHECKPOINT_GOSSIP_SYNC_AUDIT_OPTION, + registry_id: str = _CHECKPOINT_REGISTRY_ID_OPTION, + source_peer_id: str = _CHECKPOINT_SOURCE_PEER_OPTION, + peer_trust_ledger_path: Path = _CHECKPOINT_PEER_TRUST_LEDGER_INPUT, + witness_trust_path: Path = _WITNESS_TRUST_INPUT, + authority_root_ledger_path: Path = _AUTHORITY_ROOT_LEDGER_INPUT, + tls_trust_path: Path = _CHECKPOINT_TLS_TRUST_INPUT, + client_peer_id: str = _CHECKPOINT_TLS_CLIENT_PEER_OPTION, + certificate_path: Path = _CHECKPOINT_TLS_CERTIFICATE_INPUT, + private_key_path: Path = _CHECKPOINT_TLS_PRIVATE_KEY_INPUT, + certificate_authority_path: Path = _CHECKPOINT_TLS_CA_INPUT, + server_hostname: str = _CHECKPOINT_TLS_SERVER_HOSTNAME_OPTION, + allowed_server_addresses: list[str] = _CHECKPOINT_TLS_ALLOWED_SERVERS_OPTION, + max_rounds: int = _CHECKPOINT_GOSSIP_SYNC_ROUNDS_OPTION, + max_records: int = _CHECKPOINT_MAX_RECORDS_OPTION, + max_attempts: int = _CHECKPOINT_GOSSIP_SYNC_ATTEMPTS_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Run the bounded resumable pull loop over pinned mutual TLS.""" + from benchmarks.agent_cli_checkpoint_registry import CheckpointRegistryStore + from benchmarks.agent_cli_gossip_sync import ( + CheckpointGossipSyncAuditStore, + CheckpointGossipSyncPolicy, + run_checkpoint_gossip_sync, + ) + from benchmarks.agent_cli_peer_trust_ledger import CheckpointPeerTrustLedger + + try: + if not registry_path.parent.exists(): + raise ValueError("checkpoint gossip registry parent must already exist") + if not sync_audit_path.parent.exists(): + raise ValueError("checkpoint gossip sync audit parent must already exist") + witness_trust, authority_ledger = _load_checkpoint_registry_dependencies( + witness_trust_path, + authority_root_ledger_path, + ) + peer_trust_ledger = CheckpointPeerTrustLedger.model_validate_json( + peer_trust_ledger_path.read_text(encoding="utf-8") + ) + client = _build_checkpoint_mtls_client( + descriptor_path=descriptor_path, + peer_trust_source=peer_trust_ledger, + tls_trust_path=tls_trust_path, + client_peer_id=client_peer_id, + certificate_path=certificate_path, + private_key_path=private_key_path, + certificate_authority_path=certificate_authority_path, + server_hostname=server_hostname, + allowed_server_addresses=allowed_server_addresses, + ) + retry_delays = tuple(min(0.05 * (2**index), 1.0) for index in range(max_attempts - 1)) + result = _run( + run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=CheckpointRegistryStore( + registry_path, + registry_id=registry_id, + ), + audit_store=CheckpointGossipSyncAuditStore( + sync_audit_path, + registry_id=registry_id, + source_peer_id=source_peer_id, + ), + peer_trust_ledger=peer_trust_ledger, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=CheckpointGossipSyncPolicy( + max_rounds=max_rounds, + max_records=max_records, + max_attempts_per_request=max_attempts, + retry_delays_seconds=retry_delays, + ), + request_sender=client, + ) + ) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI checkpoint mTLS sync failed: {exc}[/red]") + raise typer.Exit(code=1) from None + if as_json: + typer.echo(result.to_json()) + else: + console.print( + "Mutually authenticated checkpoint sync stopped " + f"reason={result.stop_reason} records={result.local_record_count} " + f"imported={result.records_imported} retries={result.retries}" + ) + + +@benchmark_app.command("agent-cli-campaign-envelope-template") +def create_agent_cli_campaign_envelope_template( + manifest_path: Path = _AGENT_CLI_MANIFEST_OPTION, + preflight_path: Path = _PREFLIGHT_INPUT_OPTION, + evidence_path: Path = _FINALIZE_EVIDENCE_OPTION, + reviews_path: Path = _FINALIZE_REVIEWS_OPTION, + review_policy_path: Path = _OPTIONAL_REVIEW_POLICY_INPUT, + reviewer_trust_path: Path = _OPTIONAL_REVIEWER_TRUST_INPUT, + reviewer_authority_path: Path = _OPTIONAL_REVIEWER_AUTHORITY_INPUT, + reviewer_enrollments_path: Path = _OPTIONAL_REVIEWER_ENROLLMENTS_INPUT, + authority_root_ledger_path: Path | None = _OPTIONAL_AUTHORITY_ROOT_LEDGER_INPUT, + attestations_path: Path = _OPTIONAL_ATTESTATIONS_INPUT, + results_path: Path = _AGENT_CLI_RESULTS_OPTION, + output_path: Path = _PREFLIGHT_OUTPUT_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Create one authority signing payload for the complete finalized campaign.""" + from benchmarks.agent_cli_adjudication import AdjudicationReviews, RecordedEvidence + from benchmarks.agent_cli_attestation import ( + ReviewAttestationBundle, + ReviewerTrustDeclaration, + build_reviewer_trust, + ) + from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + ReviewerEnrollmentBundle, + build_benchmark_authority, + build_campaign_envelope_request, + ) + from benchmarks.agent_cli_comparison import AgentCliManifest, RecordedResults + from benchmarks.agent_cli_preflight import CampaignPreflight + from benchmarks.agent_cli_review_policy import ( + ReviewerPolicyDeclaration, + build_reviewer_policy, + ) + + try: + required = { + "--review-policy": review_policy_path, + "--reviewer-trust": reviewer_trust_path, + "--reviewer-authority": reviewer_authority_path, + "--reviewer-enrollments": reviewer_enrollments_path, + "--attestations": attestations_path, + } + missing = [name for name, path in required.items() if path is None] + if missing: + raise ValueError(f"required authority artifact missing: {', '.join(missing)}") + assert review_policy_path is not None + assert reviewer_trust_path is not None + assert reviewer_authority_path is not None + assert reviewer_enrollments_path is not None + assert attestations_path is not None + manifest = AgentCliManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + preflight = CampaignPreflight.model_validate_json( + preflight_path.read_text(encoding="utf-8") + ) + evidence = RecordedEvidence.model_validate_json(evidence_path.read_text(encoding="utf-8")) + reviews = AdjudicationReviews.model_validate_json(reviews_path.read_text(encoding="utf-8")) + policy = build_reviewer_policy( + ReviewerPolicyDeclaration.model_validate_json( + review_policy_path.read_text(encoding="utf-8") + ) + ) + authority = build_benchmark_authority( + BenchmarkAuthorityDeclaration.model_validate_json( + reviewer_authority_path.read_text(encoding="utf-8") + ) + ) + trust = build_reviewer_trust( + ReviewerTrustDeclaration.model_validate_json( + reviewer_trust_path.read_text(encoding="utf-8") + ), + policy, + ) + enrollments = ReviewerEnrollmentBundle.model_validate_json( + reviewer_enrollments_path.read_text(encoding="utf-8") + ) + attestations = ReviewAttestationBundle.model_validate_json( + attestations_path.read_text(encoding="utf-8") + ) + results = RecordedResults.model_validate_json(results_path.read_text(encoding="utf-8")) + authority_root_ledger = None + if authority_root_ledger_path is not None: + from benchmarks.agent_cli_transparency import SignedAuthorityRootLedger + + authority_root_ledger = SignedAuthorityRootLedger.model_validate_json( + authority_root_ledger_path.read_text(encoding="utf-8") + ) + template = build_campaign_envelope_request( + authority=authority, + manifest=manifest, + preflight=preflight, + evidence=evidence, + reviews=reviews, + review_policy=policy, + reviewer_trust=trust, + reviewer_enrollments=enrollments, + attestations=attestations, + results=results, + authority_root_ledger=authority_root_ledger, + ) + if not output_path.parent.exists(): + raise ValueError("campaign envelope output parent must already exist") + if output_path.exists(): + raise ValueError("campaign envelope output already exists") + _write_new_evidence(output_path, template.to_json()) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI campaign envelope template failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(template.to_json()) + else: + console.print("Created authority campaign envelope signing request") + + +@benchmark_app.command("agent-cli-status") +def show_agent_cli_campaign_status( + manifest_path: Path = _AGENT_CLI_MANIFEST_OPTION, + preflight_path: Path | None = _OPTIONAL_PREFLIGHT_INPUT, + evidence_path: Path | None = _STATUS_EVIDENCE_OPTION, + reviews_path: Path | None = _STATUS_REVIEWS_OPTION, + results_path: Path | None = _STATUS_RESULTS_OPTION, + review_policy_path: Path | None = _OPTIONAL_REVIEW_POLICY_INPUT, + reviewer_trust_path: Path | None = _OPTIONAL_REVIEWER_TRUST_INPUT, + attestations_path: Path | None = _OPTIONAL_ATTESTATIONS_INPUT, + reviewer_authority_path: Path | None = _OPTIONAL_REVIEWER_AUTHORITY_INPUT, + reviewer_enrollments_path: Path | None = _OPTIONAL_REVIEWER_ENROLLMENTS_INPUT, + campaign_envelope_path: Path | None = _OPTIONAL_CAMPAIGN_ENVELOPE_INPUT, + authority_root_ledger_path: Path | None = _OPTIONAL_AUTHORITY_ROOT_LEDGER_INPUT, + transparency_proof_path: Path | None = _OPTIONAL_TRANSPARENCY_PROOF_INPUT, + transparency_consistency_path: Path | None = (_OPTIONAL_TRANSPARENCY_CONSISTENCY_INPUT), + transparency_witness_trust_path: Path | None = (_OPTIONAL_TRANSPARENCY_WITNESS_TRUST_INPUT), + witness_checkpoint_path: Path | None = _OPTIONAL_WITNESS_CHECKPOINT_INPUT, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Validate and report campaign lifecycle artifacts without executing commands.""" + from benchmarks.agent_cli_adjudication import AdjudicationReviews, RecordedEvidence + from benchmarks.agent_cli_campaign import build_campaign_status + from benchmarks.agent_cli_comparison import AgentCliManifest, RecordedResults + from benchmarks.agent_cli_preflight import CampaignPreflight, ReviewTemplate + from benchmarks.agent_cli_review_policy import ( + ReviewerPolicyDeclaration, + build_reviewer_policy, + ) + + try: + manifest = AgentCliManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + preflight = ( + CampaignPreflight.model_validate_json(preflight_path.read_text(encoding="utf-8")) + if preflight_path is not None + else None + ) + evidence = ( + RecordedEvidence.model_validate_json(evidence_path.read_text(encoding="utf-8")) + if evidence_path is not None + else None + ) + review_template = None + reviews = None + if reviews_path is not None: + review_payload = json.loads(reviews_path.read_text(encoding="utf-8")) + if not isinstance(review_payload, dict): + raise ValueError("reviews JSON must be an object") + if review_payload.get("review_completed") is False: + review_template = ReviewTemplate.model_validate(review_payload) + else: + reviews = AdjudicationReviews.model_validate(review_payload) + results = ( + RecordedResults.model_validate_json(results_path.read_text(encoding="utf-8")) + if results_path is not None + else None + ) + review_policy = None + reviewer_trust = None + attestations = None + reviewer_authority = None + reviewer_enrollments = None + campaign_envelope = None + authority_root_ledger = None + transparency_proof = None + transparency_consistency_proof = None + transparency_witness_trust = None + witness_checkpoint = None + if review_policy_path is not None: + declaration = ReviewerPolicyDeclaration.model_validate_json( + review_policy_path.read_text(encoding="utf-8") + ) + review_policy = build_reviewer_policy(declaration) + if reviewer_trust_path is not None: + if review_policy is None: + raise ValueError("--review-policy is required with --reviewer-trust") + from benchmarks.agent_cli_attestation import ( + ReviewerTrustDeclaration, + build_reviewer_trust, + ) + + reviewer_trust = build_reviewer_trust( + ReviewerTrustDeclaration.model_validate_json( + reviewer_trust_path.read_text(encoding="utf-8") + ), + review_policy, + ) + if attestations_path is not None: + from benchmarks.agent_cli_attestation import ReviewAttestationBundle + + attestations = ReviewAttestationBundle.model_validate_json( + attestations_path.read_text(encoding="utf-8") + ) + if reviewer_authority_path is not None: + from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + build_benchmark_authority, + ) + + reviewer_authority = build_benchmark_authority( + BenchmarkAuthorityDeclaration.model_validate_json( + reviewer_authority_path.read_text(encoding="utf-8") + ) + ) + if reviewer_enrollments_path is not None: + from benchmarks.agent_cli_authority import ReviewerEnrollmentBundle + + reviewer_enrollments = ReviewerEnrollmentBundle.model_validate_json( + reviewer_enrollments_path.read_text(encoding="utf-8") + ) + if campaign_envelope_path is not None: + from benchmarks.agent_cli_authority import SignedCampaignEnvelope + + campaign_envelope = SignedCampaignEnvelope.model_validate_json( + campaign_envelope_path.read_text(encoding="utf-8") + ) + if authority_root_ledger_path is not None: + from benchmarks.agent_cli_transparency import SignedAuthorityRootLedger + + authority_root_ledger = SignedAuthorityRootLedger.model_validate_json( + authority_root_ledger_path.read_text(encoding="utf-8") + ) + if transparency_proof_path is not None: + from benchmarks.agent_cli_transparency import TransparencyInclusionProof + + transparency_proof = TransparencyInclusionProof.model_validate_json( + transparency_proof_path.read_text(encoding="utf-8") + ) + if transparency_consistency_path is not None: + from benchmarks.agent_cli_transparency import ( + TransparencyConsistencyProof, + ) + + transparency_consistency_proof = TransparencyConsistencyProof.model_validate_json( + transparency_consistency_path.read_text(encoding="utf-8") + ) + if transparency_witness_trust_path is not None: + from benchmarks.agent_cli_witness import ( + TransparencyWitnessTrustDeclaration, + build_transparency_witness_trust, + ) + + transparency_witness_trust = build_transparency_witness_trust( + TransparencyWitnessTrustDeclaration.model_validate_json( + transparency_witness_trust_path.read_text(encoding="utf-8") + ) + ) + if witness_checkpoint_path is not None: + from benchmarks.agent_cli_witness import SignedWitnessCheckpoint + + witness_checkpoint = SignedWitnessCheckpoint.model_validate_json( + witness_checkpoint_path.read_text(encoding="utf-8") + ) + status = build_campaign_status( + manifest, + preflight=preflight, + evidence=evidence, + review_template=review_template, + reviews=reviews, + results=results, + review_policy=review_policy, + reviewer_trust=reviewer_trust, + attestations=attestations, + reviewer_authority=reviewer_authority, + reviewer_enrollments=reviewer_enrollments, + campaign_envelope=campaign_envelope, + authority_root_ledger=authority_root_ledger, + transparency_proof=transparency_proof, + transparency_consistency_proof=transparency_consistency_proof, + transparency_witness_trust=transparency_witness_trust, + witness_checkpoint=witness_checkpoint, + ) + except (OSError, ValueError) as exc: + console.print(f"[red]Agent CLI campaign status failed: {exc}[/red]") + raise typer.Exit(code=1) from None + + if as_json: + typer.echo(status.to_json()) + else: + console.print( + f"Campaign stage={status.stage.value} next={status.next_action} " + "paid_execution_authorized=false" + ) + + +@benchmark_app.command("agent-cli-rehearse") +def rehearse_agent_cli_trials( + output_dir: Path = _REHEARSAL_OUTPUT_OPTION, + source_root: Path | None = _RECORDER_SOURCE_OPTION, + worktree_root: Path | None = _RECORDER_WORKTREE_OPTION, + revision: str = _REHEARSAL_REVISION_OPTION, + as_json: bool = _AGENT_CLI_JSON_OPTION, +) -> None: + """Run the complete three-arm evidence pipeline with internal zero-cost fixtures.""" + from benchmarks.agent_cli_rehearsal import ( + publish_local_rehearsal, + resolve_git_revision, + run_local_rehearsal, + ) + + source = (source_root or Path.cwd()).resolve() + output = output_dir.resolve() + if output.exists(): + console.print(f"[red]Rehearsal output already exists: {output}[/red]") + raise typer.Exit(code=1) + if not output.parent.exists(): + console.print("[red]Rehearsal output parent must already exist[/red]") + raise typer.Exit(code=1) + + temporary_root: tempfile.TemporaryDirectory[str] | None = None + if worktree_root is None: + temporary_root = tempfile.TemporaryDirectory(prefix="morphic-agent-cli-rehearsal-") + isolated_root = Path(temporary_root.name) + else: + isolated_root = worktree_root.resolve() + try: + pinned_revision = _run(resolve_git_revision(source, revision)) + artifacts = _run( + run_local_rehearsal( + source_root=source, + worktree_root=isolated_root, + workspace_revision=pinned_revision, + python_executable=sys.executable, + ) + ) + publish_local_rehearsal(output, artifacts) + except (OSError, RuntimeError, ValueError) as exc: + console.print(f"[red]Agent CLI rehearsal failed: {exc}[/red]") + raise typer.Exit(code=1) from None + finally: + if temporary_root is not None: + temporary_root.cleanup() + + if as_json: + typer.echo( + json.dumps( + artifacts.results.model_dump(mode="json"), + ensure_ascii=False, + sort_keys=True, + ) + ) + else: + console.print( + f"Rehearsed {len(artifacts.results.observations)} local trials " + f"at $0.000000 into {output}" + ) diff --git a/interface/cli/commands/chat_control.py b/interface/cli/commands/chat_control.py new file mode 100644 index 0000000..d062d53 --- /dev/null +++ b/interface/cli/commands/chat_control.py @@ -0,0 +1,120 @@ +"""morphic chat-control — inspect or cancel an opt-in active chat turn.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import typer + +from interface.cli._utils import _run +from interface.cli.chat_control_transport import ( + discover_active_chat_sessions, + send_chat_control_command, +) +from interface.cli.formatters import console + +chat_control_app = typer.Typer() +_WORKSPACE_OPTION = typer.Option(None, "--workspace", help="Workspace root.") +_SESSION_OPTION = typer.Option(None, "--session", help="Morphic chat session id.") + + +@chat_control_app.command("status") +def status_cmd( + session_id: str | None = _SESSION_OPTION, + json_output: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), + workspace: Path | None = _WORKSPACE_OPTION, +) -> None: + """Report whether one opt-in chat session has an active turn.""" + _run_control_command( + command="status", + session_id=session_id, + json_output=json_output, + workspace_root=workspace or Path.cwd(), + prompt=None, + ) + + +@chat_control_app.command("cancel") +def cancel_cmd( + session_id: str | None = _SESSION_OPTION, + json_output: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), + workspace: Path | None = _WORKSPACE_OPTION, +) -> None: + """Cancel one active turn without terminating its chat REPL.""" + payload = _run_control_command( + command="cancel", + session_id=session_id, + json_output=json_output, + workspace_root=workspace or Path.cwd(), + prompt=None, + ) + if payload.get("cancelled") is not True: + raise typer.Exit(code=1) + + +@chat_control_app.command("steer") +def steer_cmd( + prompt: str = typer.Argument(..., help="Replacement prompt for the native session."), + session_id: str | None = _SESSION_OPTION, + json_output: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), + workspace: Path | None = _WORKSPACE_OPTION, +) -> None: + """Cancel the active turn and continue with a replacement prompt.""" + payload = _run_control_command( + command="steer", + session_id=session_id, + json_output=json_output, + workspace_root=workspace or Path.cwd(), + prompt=prompt, + ) + if payload.get("steered") is not True: + raise typer.Exit(code=1) + + +def _run_control_command( + *, + command: str, + session_id: str | None, + json_output: bool, + workspace_root: Path, + prompt: str | None, +) -> dict[str, object]: + try: + resolved_session = session_id or _discover_single_session(workspace_root) + payload = _run( + send_chat_control_command( + workspace_root=workspace_root, + session_id=resolved_session, + command=command, + prompt=prompt, + ) + ) + except (OSError, RuntimeError, ValueError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from None + + if json_output: + typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + elif command == "status": + console.print( + f"session={payload['session_id']} active_turn={str(payload['active_turn']).lower()}" + ) + elif command == "cancel": + console.print( + f"session={payload['session_id']} cancelled={str(payload['cancelled']).lower()}" + ) + else: + console.print( + f"session={payload['session_id']} steered={str(payload['steered']).lower()}" + ) + return payload + + +def _discover_single_session(workspace_root: Path) -> str: + sessions = discover_active_chat_sessions(workspace_root=workspace_root) + if not sessions: + raise ValueError("no active chat control session; start chat with --control") + if len(sessions) > 1: + raise ValueError("multiple active chat control sessions; specify --session") + return sessions[0] diff --git a/interface/cli/commands/hooks.py b/interface/cli/commands/hooks.py new file mode 100644 index 0000000..35f4a24 --- /dev/null +++ b/interface/cli/commands/hooks.py @@ -0,0 +1,126 @@ +"""morphic hooks — validate and run Morphic Chat hooks.""" + +from __future__ import annotations + +import json +import uuid +from pathlib import Path + +import typer + +from application.use_cases.execute_chat_hook import ExecuteChatHookUseCase +from domain.entities.chat_session import ChatSession, PermissionMode +from domain.entities.hook import HookType +from infrastructure.chat.jsonl_session_store import JsonlChatSessionStore +from infrastructure.hooks.workspace_hook_registry import WorkspaceHookRegistry +from interface.cli._utils import _run +from interface.cli.chat_command import _chat_hook_execution_mode, _chat_hook_executor +from interface.cli.formatters import console + +hooks_app = typer.Typer() +_WORKSPACE_OPTION = typer.Option( + None, + "--workspace", + help="Workspace root.", +) + + +@hooks_app.command("run") +def run_hook( + hook_type: str = typer.Argument(..., help="Hook type to run, e.g. pre_tool."), + json_output: bool = typer.Option( + False, + "--json", + help="Emit machine-readable JSON.", + ), + workspace: Path | None = _WORKSPACE_OPTION, +) -> None: + """Run hooks of one type through the configured hook executor.""" + try: + parsed_hook_type = HookType(hook_type) + payload = _run( + _run_hook_payload( + hook_type=parsed_hook_type, + workspace_root=workspace or Path.cwd(), + ) + ) + except ValueError as exc: + message = str(exc) + if "is not a valid HookType" in message: + valid = ", ".join(item.value for item in HookType) + message = f"Invalid hook type '{hook_type}'. Expected one of: {valid}" + typer.echo(f"Error: {message}", err=True) + raise typer.Exit(code=2) from None + + if json_output: + typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + else: + summary = payload["summary"] + console.print( + "hooks " + f"type={payload['hook_type']} " + f"mode={payload['hook_execution_mode']} " + f"succeeded={summary['succeeded']} " + f"failed={summary['failed']} " + f"skipped={summary['skipped']}" + ) + + if payload["summary"]["failed"] > 0: + raise typer.Exit(code=1) + + +async def _run_hook_payload( + *, + hook_type: HookType, + workspace_root: Path, +) -> dict[str, object]: + store = JsonlChatSessionStore(workspace_root=workspace_root) + session = ChatSession.start( + session_id=uuid.uuid4().hex[:12], + goal=f"run {hook_type.value} hooks", + permission_mode=PermissionMode.CONFIRM_DESTRUCTIVE, + ) + hook_runner = ExecuteChatHookUseCase( + session_store=store, + hook_registry=WorkspaceHookRegistry(workspace_root), + hook_executor=_chat_hook_executor(workspace_root=workspace_root), + ) + result = await hook_runner.execute(session=session, hook_type=hook_type) + hook_results = [ + { + "exit_code": hook_result.exit_code, + "request_id": hook_result.request_id, + "stderr_summary": hook_result.stderr_summary, + "stdout_summary": hook_result.stdout_summary, + "success": hook_result.success, + } + for hook_result in result.hook_results + ] + completed_by_request_id = { + event.payload["request_id"]: event + for event in result.events + if event.type.value == "hook_execution_completed" + } + for hook_result in hook_results: + event = completed_by_request_id.get(hook_result["request_id"]) + if event is not None: + hook_result["hook_name"] = event.payload["hook_name"] + + skipped = sum(1 for event in result.events if event.type.value == "hook_execution_skipped") + failed = sum(1 for hook_result in result.hook_results if not hook_result.success) + succeeded = sum(1 for hook_result in result.hook_results if hook_result.success) + return { + "diagnostics": [ + diagnostic.model_dump(mode="json") for diagnostic in result.diagnostics + ], + "events": [event.model_dump(mode="json") for event in result.events], + "hook_execution_mode": _chat_hook_execution_mode(), + "hook_type": hook_type.value, + "results": hook_results, + "session_id": result.session.id, + "summary": { + "failed": failed, + "skipped": skipped, + "succeeded": succeeded, + }, + } diff --git a/interface/cli/main.py b/interface/cli/main.py index d40f45c..a0d44cc 100644 --- a/interface/cli/main.py +++ b/interface/cli/main.py @@ -49,6 +49,7 @@ def _register_commands() -> None: from interface.cli.chat_command import chat_cmd, code_cmd from interface.cli.commands.a2a import a2a_app from interface.cli.commands.benchmark import benchmark_app + from interface.cli.commands.chat_control import chat_control_app from interface.cli.commands.cognitive import cognitive_app from interface.cli.commands.context import context_app from interface.cli.commands.cost import cost_app @@ -57,6 +58,7 @@ def _register_commands() -> None: from interface.cli.commands.engine import engine_app from interface.cli.commands.evolution import evolution_app from interface.cli.commands.fallback import fallback_app + from interface.cli.commands.hooks import hooks_app from interface.cli.commands.learning import learning_app from interface.cli.commands.marketplace import marketplace_app from interface.cli.commands.mcp import mcp_app @@ -68,12 +70,18 @@ def _register_commands() -> None: app.command("chat")(chat_cmd) app.command("code")(code_cmd) + app.add_typer( + chat_control_app, + name="chat-control", + help="Inspect or cancel an opt-in active chat turn.", + ) app.add_typer(task_app, name="task", help="Create, list, show, and cancel tasks.") app.add_typer(plan_app, name="plan", help="Create, review, approve, and reject plans.") app.add_typer(model_app, name="model", help="Manage LLM models.") app.add_typer(cost_app, name="cost", help="View cost tracking and budget.") app.add_typer(mcp_app, name="mcp", help="Manage MCP server.") app.add_typer(engine_app, name="engine", help="Manage agent execution engines.") + app.add_typer(hooks_app, name="hooks", help="Validate and run Morphic Chat hooks.") app.add_typer( council_app, name="council", @@ -109,7 +117,7 @@ def _register_commands() -> None: app.add_typer( benchmark_app, name="benchmark", - help="Run UCL benchmarks (context continuity, dedup accuracy).", + help="Run UCL and recorded agent CLI benchmarks.", ) app.add_typer( a2a_app, diff --git a/interface/cli/native_event_progress.py b/interface/cli/native_event_progress.py new file mode 100644 index 0000000..fa6f9d1 --- /dev/null +++ b/interface/cli/native_event_progress.py @@ -0,0 +1,50 @@ +"""Concise terminal rendering for normalized native-engine events.""" + +from __future__ import annotations + +from collections.abc import Callable + +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType +from domain.ports.agent_engine import AgentEngineEventSinkPort +from interface.cli.formatters import console + +_LABELS = { + AgentEngineEventType.RUN_STARTED: "started", + AgentEngineEventType.TOOL_STARTED: "tool started", + AgentEngineEventType.TOOL_COMPLETED: "tool completed", + AgentEngineEventType.FILE_CHANGED: "file changed", + AgentEngineEventType.PLAN_UPDATED: "plan updated", + AgentEngineEventType.RUN_COMPLETED: "completed", + AgentEngineEventType.RUN_FAILED: "failed", + AgentEngineEventType.ERROR: "error", +} +_DETAIL_TYPES = { + AgentEngineEventType.TOOL_STARTED, + AgentEngineEventType.TOOL_COMPLETED, + AgentEngineEventType.FILE_CHANGED, + AgentEngineEventType.PLAN_UPDATED, + AgentEngineEventType.RUN_FAILED, + AgentEngineEventType.ERROR, +} + + +class NativeEventProgressRenderer(AgentEngineEventSinkPort): + """Render selected lifecycle events without exposing raw provider payloads.""" + + def __init__(self, printer: Callable[[str], None] | None = None) -> None: + self._printer = printer or console.print + + async def publish(self, event: AgentEngineEvent) -> None: + label = _LABELS.get(event.type) + if label is None: + return + line = f"{event.engine.value} | {label}" + if event.type in _DETAIL_TYPES and event.text: + line = f"{line}: {self._compact(event.text)}" + self._printer(line) + + def _compact(self, value: str, limit: int = 160) -> str: + text = " ".join(value.split()) + if len(text) <= limit: + return text + return f"{text[: limit - 1]}…" diff --git a/interface/cli/turn_control.py b/interface/cli/turn_control.py new file mode 100644 index 0000000..e3b4f77 --- /dev/null +++ b/interface/cli/turn_control.py @@ -0,0 +1,107 @@ +"""Signal-aware control for one active interactive chat turn.""" + +from __future__ import annotations + +import asyncio +import signal +from collections.abc import Callable, Coroutine, Iterator +from contextlib import contextmanager +from types import FrameType +from typing import Any, TypeVar + +T = TypeVar("T") + + +class TurnCancelledError(Exception): + """The interactive user cancelled the active turn without exiting the REPL.""" + + +class ActiveTurnController: + """Own at most one turn task and route Ctrl-C to that task only.""" + + def __init__(self) -> None: + self._active_task: asyncio.Task[Any] | None = None + self._cancel_requested = False + self._previous_sigint_handler: Any = None + self._steer_prompt: str | None = None + + @property + def has_active_turn(self) -> bool: + task = self._active_task + return task is not None and not task.done() + + async def run(self, operation: Callable[[], Coroutine[Any, Any, T]]) -> T: + """Run one controlled turn and distinguish requested from caller cancellation.""" + if self.has_active_turn: + raise RuntimeError("another chat turn is already active") + + self._cancel_requested = False + task = asyncio.create_task(operation()) + self._active_task = task + try: + with self._route_sigint_to_active_turn(): + try: + return await task + except asyncio.CancelledError: + if self._cancel_requested: + raise TurnCancelledError from None + raise + finally: + self._active_task = None + self._cancel_requested = False + + def cancel_active_turn(self) -> bool: + """Request cancellation without cancelling the REPL's parent task.""" + task = self._active_task + if task is None or task.done(): + return False + if not self._cancel_requested: + self._cancel_requested = True + task.cancel() + return True + + def steer_active_turn(self, prompt: str) -> bool: + """Queue one replacement prompt and cancel the active turn exactly once.""" + normalized = prompt.strip() + if not normalized: + raise ValueError("steer prompt must not be empty") + if self._steer_prompt is not None or self._cancel_requested: + return False + self._steer_prompt = normalized + if self.cancel_active_turn(): + return True + self._steer_prompt = None + return False + + def take_steer_prompt(self) -> str | None: + """Consume the pending replacement prompt after cancellation cleanup.""" + prompt = self._steer_prompt + self._steer_prompt = None + return prompt + + @contextmanager + def _route_sigint_to_active_turn(self) -> Iterator[None]: + try: + previous = signal.getsignal(signal.SIGINT) + self._previous_sigint_handler = previous + signal.signal(signal.SIGINT, self._handle_sigint) + except (OSError, ValueError): + self._previous_sigint_handler = None + yield + return + + try: + yield + finally: + signal.signal(signal.SIGINT, previous) + self._previous_sigint_handler = None + + def _handle_sigint(self, signum: int, frame: FrameType | None) -> None: + if self.cancel_active_turn(): + return + + previous = self._previous_sigint_handler + if callable(previous): + previous(signum, frame) + elif previous == signal.SIG_DFL: + signal.default_int_handler(signum, frame) diff --git a/pyproject.toml b/pyproject.toml index fa82bf9..60269a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "orjson>=3.10", "structlog>=24.4", "ddgs>=9.11.4", + "cryptography>=46.0.5", ] [project.optional-dependencies] @@ -66,7 +67,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["domain", "application", "infrastructure", "interface", "shared"] +packages = ["domain", "application", "infrastructure", "interface", "shared", "benchmarks"] [tool.ruff] target-version = "py312" diff --git a/specs/morphic-chat-cli/plan.md b/specs/morphic-chat-cli/plan.md index 2e5d802..8d09e47 100644 --- a/specs/morphic-chat-cli/plan.md +++ b/specs/morphic-chat-cli/plan.md @@ -65,6 +65,80 @@ executor role -> may use Codex CLI or local LAEE tools Keeping these separate prevents the design from hard-coding "Claude is the architect" or "Codex is always implementer." +### Direct route before always-on council + +Council is a quality mechanism, not the default execution topology for every task. The +CLI should also support a direct runtime that delegates one goal to one routed native +agent engine. Direct mode reuses `CouncilRuntimePort` temporarily so session/event +orchestration stays unchanged, but emits one `IMPLEMENTER` turn and one decision. + +Direct mode is explicit while native permission propagation is incomplete. The first +supported adapter is Codex CLI: `read-only`, `workspace-write`, and +`danger-full-access` map to explicit Codex sandboxes, while `confirm-destructive` is +rejected because `codex exec` has no interactive approval channel. Direct mode makes +exactly one route call and never converts route failure into deterministic local success. +Other native engines remain unavailable in direct mode until their permission and +workspace controls are mapped explicitly. + +Native adapter output is normalized into `AgentEngineEvent` values and attached to the +single implementer turn. `SendChatMessageUseCase` writes each value as an independent +`engine_event` before the council argument, decision, and assistant response. This makes +tool activity, file changes, plans, and lifecycle state replayable without parsing a +provider blob embedded only in the final response. Workspace and permission propagation +uses a separate `ScopedAgentEnginePort`; ordinary adapters keep the smaller common port +and are skipped when a scoped run is requested. + +For live execution, `StreamingScopedAgentEnginePort` adds an event sink without widening +the ordinary or buffered scoped contracts. Codex drains stdout and stderr concurrently, +decodes each JSONL stdout line with stateful thread/sequence tracking, and publishes it +before process completion. `StreamingCouncilRuntimePort` carries the sink to the +application layer, where the user message is persisted first and each native event is +appended immediately. The final result still retains its complete event metadata for +non-streaming consumers, but the streaming send path does not append it twice. + +Terminal progress is a best-effort observer downstream of durable event append. The +renderer uses an allowlist of lifecycle, tool, file, and plan event types, normalizes +whitespace, and truncates detail. It never reads the raw payload and ignores assistant +messages, generic progress/reasoning, and unknown provider events. A renderer exception +is logged but does not cancel native execution or roll back the already-written ledger. + +Native thread continuity is reconstructed from the Morphic ledger, not from a global +provider "last session" lookup. `ChatSession` records engine id, provider session id, +workspace root, and permission mode when a native event first identifies a thread. +Resume replays the ledger to rebuild that binding. `ResumableStreamingScopedAgentEnginePort` +is a separate capability; Codex implements it with `exec` scope flags before +`resume `. Direct runtime compares stored workspace and permission +provenance with the current turn and fails before route execution on mismatch. + +Interactive cancellation uses one interface-level `ActiveTurnController` around the +send-message coroutine. It owns exactly one child task and temporarily routes SIGINT to +that task. Controller-requested cancellation becomes a local `TurnCancelledError`, while +external cancellation of the parent remains `CancelledError`. After a controlled stop, +the REPL replays the append-only ledger before accepting another prompt so sequence and +native-session state include every event persisted during cancellation. The controller +is injectable, providing the same cancellation primitive for a later local or remote +control transport without coupling application use cases to POSIX signals. + +External local control is an opt-in interface adapter over that same controller. +`ChatControlServer` binds a random port on `127.0.0.1` only for the lifetime of an active +turn and writes a hashed, session-scoped descriptor under `.morphic/control/`. The +descriptor contains protocol version 1 and a per-server random token, with directory +mode 0700 and file mode 0600. Requests are one bounded JSON line and must match token, +session, and an allowlisted command. The client refuses any descriptor whose host is not +the exact loopback address. `morphic chat-control` is therefore a local control primitive +that can be carried through SSH or a future authenticated API/MCP bridge without opening +an unauthenticated workspace listener or changing application/domain contracts. + +Steering is provider-neutral control built on cancellation and explicit resume. The +loopback protocol accepts one non-empty replacement prompt capped at 2048 UTF-8 bytes. +`ActiveTurnController` uses first-writer-wins queueing, cancels the current child task, +and exposes the queued prompt only after cleanup. The REPL then replays the ledger, +appends `turn_steered` metadata without duplicating prompt content, and sends the prompt +through the ordinary message use case. Replay restores the provider-native session id, +workspace, and permission provenance before routing, so steer cannot switch providers or +widen scope. Queued prompts bypass local slash-command parsing and are always native +session input. + ### Morphic owns execution state External CLIs can contribute proposals or execute delegated tasks, but Morphic should own: @@ -313,4 +387,5 @@ Manual validation: 8. Add `morphic code ""` one-shot entry. 9. Add diagnostics commands. 10. Add first external engine adapter behind registry. - +11. Add an explicit single-engine direct route before making real execution the default. +12. Add normalized streaming/resume/approval events and native permission mappings. diff --git a/specs/morphic-chat-cli/spec.md b/specs/morphic-chat-cli/spec.md index a2ec4e4..ddae71a 100644 --- a/specs/morphic-chat-cli/spec.md +++ b/specs/morphic-chat-cli/spec.md @@ -100,6 +100,21 @@ Acceptance criteria: - **FR-13:** The CLI shall support permission modes compatible with Morphic policy: `read-only`, `workspace-write`, `confirm-destructive`, `danger-full-access`. - **FR-14:** The CLI shall support structured non-interactive output for diagnostics: `morphic chat --doctor --json` or equivalent. - **FR-15:** The system shall resume the latest session with `morphic chat --resume latest`. +- **FR-16:** The CLI shall offer an explicit single-engine direct route that invokes `RouteToEngineUseCase` exactly once per user turn instead of forcing planner/critic/leader council execution. +- **FR-17:** The direct route shall accept an optional preferred engine id and shall surface route failure or empty output instead of silently returning a deterministic local response as success. +- **FR-18:** Direct external-engine execution shall propagate workspace root and Morphic permission mode to a native adapter that can preserve them. Codex direct mode maps `read-only`, `workspace-write`, and `danger-full-access` to explicit Codex sandboxes and rejects `confirm-destructive`, because non-interactive execution cannot surface a new approval prompt. +- **FR-19:** Native JSONL engine output shall be normalized into provider-independent engine events while retaining the raw provider payload for audit and forward compatibility. +- **FR-20:** A direct native run shall persist each normalized engine event as an independent append-only session event, in provider order and before the corresponding council argument and assistant response. +- **FR-21:** Workspace root and permission mode shall be passed only through adapters that explicitly implement scoped execution; routing shall skip unsupported adapters rather than silently discarding either control. +- **FR-22:** A streaming native adapter shall publish normalized events and append them to the session ledger before the native process exits; final buffered metadata shall not duplicate events already persisted through the stream. +- **FR-23:** The terminal shall render a concise allowlist of live native lifecycle, tool, file, and plan events after durable append. It shall not render raw provider payloads, hidden reasoning, or duplicate the final assistant message as progress. +- **FR-24:** Native session resume shall bind the provider session id to its engine, original workspace root, and permission mode. Resume shall use an explicit resumable adapter capability and shall fail closed when any safety provenance differs or is absent. +- **FR-25:** Claude Code direct mode shall preserve its native project harness while normalizing `stream-json` init, assistant, tool use/result, and final result messages into the same Morphic event and scoped-resume contracts as Codex. +- **FR-26:** A native resume session id shall be pinned to its originating engine. Routing shall never pass a Claude session id to Codex, a Codex thread id to Claude, or any provider-native id to a fallback engine. +- **FR-27:** Cancelling a streaming turn shall append a `turn_cancelled` event after any already-delivered native events, preserve the original cancellation signal, and avoid presenting a cancelled turn as a completed assistant response. A cancellation not handled by an interactive control surface shall terminate the CLI with exit code 130. +- **FR-28:** During an active `morphic chat` turn, Ctrl-C shall cancel only that turn, allow provider cleanup and durable cancellation recording to finish, rebuild the in-memory session from the ledger, and return to the prompt. Cancellation of the outer REPL task and Ctrl-C while no turn is active shall retain normal process-level cancellation behavior. +- **FR-29:** The CLI shall optionally expose an active chat turn through an authenticated, session-scoped loopback control transport. It shall bind only to `127.0.0.1`, use a random token stored in a mode-0600 descriptor, reject token/session/command mismatches, remove the descriptor after the turn, and remain disabled unless `morphic chat --control` is supplied. `morphic chat-control status/cancel` shall use this transport without directly exposing provider process details. +- **FR-30:** `morphic chat-control steer` shall accept one non-empty replacement prompt of at most 2048 UTF-8 bytes, reject later steer requests while cancellation cleanup is pending, cancel the current turn, replay the ledger, append a `turn_steered` audit event, and submit the replacement as a normal message in the same provider-bound native session and safety scope. A replacement beginning with `/` shall not be interpreted as a local slash command. ## Non-Functional Requirements @@ -123,6 +138,8 @@ Acceptance criteria: | Framework imports in new domain files | 0 | | Existing unit suite regression | 0 failures | | User-visible first progress after submit | <= 500 ms in local dev | +| Direct-route engine calls per turn | exactly 1 | +| Direct-route failures hidden by local fallback | 0 | ## Relationship to Existing Specs @@ -138,4 +155,3 @@ Acceptance criteria: - [x] LAEE risk classification declared. - [x] Unit + integration test strategy defined. - [x] Ollama path included. - diff --git a/specs/morphic-chat-cli/tasks.md b/specs/morphic-chat-cli/tasks.md index 05e51e9..f745213 100644 --- a/specs/morphic-chat-cli/tasks.md +++ b/specs/morphic-chat-cli/tasks.md @@ -156,15 +156,449 @@ - [x] T1504 Preserve existing tool execution behavior when no hook planner is injected. - [x] T1505 Preserve append-only session ledger ordering across hook/tool/verification events. +## Phase 16 - Hook Execution Use Case + +- [x] T1601 Add hook execution request/result domain models. +- [x] T1602 Add `HookExecutorPort` for approved hook command execution. +- [x] T1603 Add `hook_execution_requested` and `hook_execution_completed` ledger events. +- [x] T1604 Add `ExecuteChatHookUseCase` that executes enabled hooks through the port. +- [x] T1605 Record disabled hooks as skipped without calling the executor. +- [x] T1606 Block hook execution when hook diagnostics contain FAIL results. +- [x] T1607 Keep shell-backed hook execution infrastructure deferred until approval/risk policy is wired. + +## Phase 17 - Hook Runner Wiring + +- [x] T1701 Add a no-op `HookExecutorPort` infrastructure adapter for safe wiring. +- [x] T1702 Allow `ExecuteChatToolUseCase` to accept an optional `ExecuteChatHookUseCase`. +- [x] T1703 Execute `pre_tool` hooks before tool execution when a hook runner is injected. +- [x] T1704 Execute `post_tool` hooks after tool execution when a hook runner is injected. +- [x] T1705 Preserve existing hook planning behavior when only a hook planner is injected. +- [x] T1706 Preserve append-only session ledger ordering across hook execution/tool/verification events. +- [x] T1707 Keep real shell-backed hook command execution deferred. + +## Phase 18 - Shell-backed Hook Executor + +- [x] T1801 Add `ShellHookExecutor` that maps hook commands to LAEE `shell_exec` actions. +- [x] T1802 Run hook shell commands with workspace root as `cwd` and configurable timeout. +- [x] T1803 Normalize LAEE success observations into successful hook execution results. +- [x] T1804 Normalize LAEE denied/error observations into failed hook execution results. +- [x] T1805 Stop tool execution when an injected `pre_tool` hook runner records a failed hook result. +- [x] T1806 Preserve post-tool hook failure as ledger data without adding rollback behavior. + +## Phase 19 - Hook Execution Mode Wiring + +- [x] T1901 Add chat hook executor factory with safe no-op default. +- [x] T1902 Select shell-backed hook execution only when `MORPHIC_CHAT_HOOK_EXECUTION=shell`. +- [x] T1903 Reject unknown hook execution modes with user-facing validation. +- [x] T1904 Build shell hook executor with LAEE local executor settings when opt-in is enabled. +- [x] T1905 Surface hook execution mode in `morphic chat --doctor --json`. + +## Phase 20 - Manual Hook Run CLI + +- [x] T2001 Add `morphic hooks run ` for explicit hook execution. +- [x] T2002 Persist manual hook execution events to `.morphic/sessions/*.jsonl`. +- [x] T2003 Emit JSON output for automation via `morphic hooks run --json`. +- [x] T2004 Preserve safe no-op default for manual hook runs. +- [x] T2005 Validate shell opt-in execution with `MORPHIC_CHAT_HOOK_EXECUTION=shell`. +- [x] T2006 Confirm shell opt-in writes LAEE audit log entries. + +## Phase 21 - REPL Hook Run UX + +- [x] T2101 Add `/hooks run ` handling inside `morphic chat`. +- [x] T2102 Record REPL hook run slash commands in the current chat session ledger. +- [x] T2103 Record REPL hook execution events in the current chat session ledger. +- [x] T2104 Preserve no-op default for REPL hook runs. +- [x] T2105 Respect `MORPHIC_CHAT_HOOK_EXECUTION=shell` for REPL hook runs. +- [x] T2106 Validate shell opt-in REPL hook execution writes LAEE audit log entries. + +## Phase 22 - REPL Tool Run No-op UX + +- [x] T2201 Add `NoopToolExecutor` as a safe default chat tool executor. +- [x] T2202 Add `/tools run [json_arguments]` handling inside `morphic chat`. +- [x] T2203 Record REPL tool run slash commands in the current chat session ledger. +- [x] T2204 Route REPL tool runs through `ExecuteChatToolUseCase`. +- [x] T2205 Inject existing hook runner flow around REPL tool runs. +- [x] T2206 Preserve no-op default so explicit `/tools run` does not mutate the workspace. +- [x] T2207 Reject invalid JSON tool arguments with a user-facing message. + +## Phase 23 - REPL Tool Run LAEE Opt-In + +- [x] T2301 Add chat tool executor factory with safe no-op default. +- [x] T2302 Select LAEE-backed tool execution only when `MORPHIC_CHAT_TOOL_EXECUTION=laee`. +- [x] T2303 Reject unknown tool execution modes with user-facing validation. +- [x] T2304 Build LAEE tool executor with shared local executor settings when opt-in is enabled. +- [x] T2305 Surface tool execution mode in `morphic chat --doctor --json`. +- [x] T2306 Validate REPL `/tools run shell_exec ...` writes LAEE audit log entries when opted in. + +## Phase 24 - REPL Tool Run Failure Reporting + +- [x] T2401 Report LAEE denied/error tool results with `success=False` and `exit_code`. +- [x] T2402 Surface LAEE failure stderr summaries in `/tools run` output. +- [x] T2403 Assess `/tools run` risk from tool name and JSON arguments before execution. +- [x] T2404 Validate denied destructive tools keep the target file intact and write audit logs. + +## Phase 25 - CLI Permission Mode Controls + +- [x] T2501 Add `--permission-mode` to `morphic chat`. +- [x] T2502 Add `--permission-mode` to `morphic code`. +- [x] T2503 Persist selected permission mode in session start ledger events. +- [x] T2504 Surface selected permission mode through `/status`. +- [x] T2505 Keep read-only tool blocking user-facing instead of crashing the REPL. + +## Phase 26 - Single-Engine Direct Route + +- [x] T2601 Add RED tests for one-call route-backed direct runtime behavior. +- [x] T2602 Add `RouteChatDirectRuntime` and normalize its result as one implementer turn. +- [x] T2603 Add `--route-direct` to `morphic chat` and `morphic code`. +- [x] T2604 Add optional `--engine ` preference for direct mode. +- [x] T2605 Reject simultaneous `--route-direct` and `--route-council` modes. +- [x] T2606 Surface route failure and empty output without local success fallback. +- [x] T2607 Require explicit `danger-full-access` until native permission mapping exists. +- [x] T2608 Keep external engines fake-only in unit tests. + +## Phase 27 - Codex Permission and JSONL Normalization + +- [x] T2701 Add strict provider-independent native engine event entities. +- [x] T2702 Parse Codex JSONL thread, turn, item, completion, failure, and error records. +- [x] T2703 Extract Codex thread id, final assistant message, usage, and parse diagnostics. +- [x] T2704 Replace deprecated `--full-auto` with explicit `--sandbox`. +- [x] T2705 Map read-only, workspace-write, and danger-full-access permissions to Codex. +- [x] T2706 Reject confirm-destructive because non-interactive Codex cannot prompt. +- [x] T2707 Pass workspace root with Codex `--cd`. +- [x] T2708 Restrict direct route to explicit `--engine codex_cli` until other mappings exist. +- [x] T2709 Preserve legacy single-JSON/raw-output parsing compatibility. + +## Phase 28 - Native Event Ledger and Scoped Execution Contract + +- [x] T2801 Attach normalized native engine events to direct-runtime turns. +- [x] T2802 Persist each native engine event as an append-only `engine_event` chat event. +- [x] T2803 Preserve native event order before the corresponding council argument. +- [x] T2804 Add a separate `ScopedAgentEnginePort` for workspace and permission-aware execution. +- [x] T2805 Keep the common `AgentEnginePort` compatible with existing engine adapters. +- [x] T2806 Skip permission-unaware engines instead of silently dropping scoped controls. +- [x] T2807 Retain raw provider payloads in the session ledger for audit and replay. + +## Phase 29 - Incremental Codex Event Streaming + +- [x] T2901 Add an async subprocess runner that drains stdout and stderr concurrently. +- [x] T2902 Deliver decoded stdout lines before native process completion. +- [x] T2903 Add a stateful Codex JSONL event decoder for sequence and thread propagation. +- [x] T2904 Add explicit streaming scoped-engine and council runtime capability ports. +- [x] T2905 Route streaming scoped requests only to adapters that implement the capability. +- [x] T2906 Persist user input before execution and native events as they arrive. +- [x] T2907 Avoid replaying streamed result metadata into duplicate ledger events. +- [x] T2908 Preserve buffered execution for non-streaming callers and adapters. + +## Phase 30 - Live Native Progress Rendering + +- [x] T3001 Add an optional native event observer to send-message orchestration. +- [x] T3002 Publish to the observer only after durable ledger append succeeds. +- [x] T3003 Keep observer failures best-effort so presentation cannot erase audit state. +- [x] T3004 Add a concise terminal renderer for selected lifecycle/tool/file/plan events. +- [x] T3005 Suppress assistant-message, unknown, raw payload, and reasoning content. +- [x] T3006 Compact whitespace and cap rendered event detail length. +- [x] T3007 Wire the renderer into Chat REPL and one-shot code streaming paths. + +## Phase 31 - Scoped Codex Thread Resume + +- [x] T3101 Track native session id, engine, workspace, and permission provenance. +- [x] T3102 Restore native session provenance by replaying the append-only ledger. +- [x] T3103 Add a narrow resumable streaming engine capability port. +- [x] T3104 Route an explicit native session id only to resumable adapters. +- [x] T3105 Invoke `codex exec ... resume ` with explicit sandbox and cwd. +- [x] T3106 Reuse the stored Codex thread on later turns in the same Morphic session. +- [x] T3107 Refuse resume when workspace or permission provenance does not match. +- [x] T3108 Reject native resume requests missing streaming/scope context. + +## Phase 34 - Claude Native Streaming and Resume + +- [x] T3401 Normalize Claude system init, assistant, user, tool, and result JSONL. +- [x] T3402 Retain raw Claude payloads and explicit session ids in native events. +- [x] T3403 Implement scoped Claude stream-json delivery through the shared event sink. +- [x] T3404 Resume an explicit Claude session id under stored workspace/permission scope. +- [x] T3405 Preserve Claude-reported final output, model, usage, and total cost. +- [x] T3406 Allow `--engine claude_code` in route-direct Chat CLI mode. + +## Phase 35 - Provider-Pinned Native Resume + +- [x] T3501 Require the native owner engine with every resume session id. +- [x] T3502 Reject preferred-engine and resume-engine mismatches. +- [x] T3503 Skip non-owner engines before availability checks or execution. +- [x] T3504 Prevent cross-provider fallback with a provider-native session id. +- [x] T3505 Record resume-engine mismatch attempts for routing transparency. + +## Phase 36 - Durable Turn Cancellation + +- [x] T3601 Add RED tests for interrupted streaming-turn persistence. +- [x] T3602 Append `turn_cancelled` after all native events delivered before cancellation. +- [x] T3603 Re-raise the original asyncio cancellation after the ledger append. +- [x] T3604 Report Ctrl-C consistently for chat and code with exit code 130. +- [x] T3605 Verify cancellation behavior without invoking a real native CLI. + +## Phase 37 - Active Turn Control + +- [x] T3701 Add RED tests for active-turn cancellation and outer cancellation passthrough. +- [x] T3702 Add an injectable controller that owns at most one turn task. +- [x] T3703 Route SIGINT to the active child task and restore the previous handler. +- [x] T3704 Ignore repeated cancellation requests while cleanup is already running. +- [x] T3705 Replay the ledger before the interactive REPL accepts another prompt. +- [x] T3706 Preserve idle and one-shot process-level Ctrl-C behavior. +- [x] T3707 Verify continuous ledger sequencing after cancellation and REPL continuation. +- [x] T3708 Persist user input and cancellation for non-streaming runtimes too. + +## Phase 38 - Authenticated Loopback Control + +- [x] T3801 Add RED tests for external status/cancel and descriptor cleanup. +- [x] T3802 Bind a short-lived server only to `127.0.0.1` on a random port. +- [x] T3803 Write a protocol-versioned session descriptor with 0700/0600 permissions. +- [x] T3804 Authenticate requests with a random token and exact session id. +- [x] T3805 Reject non-loopback descriptors and unsupported commands fail closed. +- [x] T3806 Add explicit `morphic chat --control` opt-in wiring. +- [x] T3807 Add `morphic chat-control status/cancel` with single-session discovery. +- [x] T3808 Remove owned descriptors after completion or cancellation cleanup. + +## Phase 39 - Provider-Neutral Steering + +- [x] T3901 Add RED tests for bounded steer queueing and native-session continuation. +- [x] T3902 Accept only non-empty replacement prompts up to 2048 UTF-8 bytes. +- [x] T3903 Make steer queue first-writer-wins during cancellation cleanup. +- [x] T3904 Add authenticated `steer` to the loopback protocol and client CLI. +- [x] T3905 Replay the ledger before submitting the replacement prompt. +- [x] T3906 Append `turn_steered` metadata before the replacement `user_message`. +- [x] T3907 Preserve provider, workspace, and permission provenance through resume. +- [x] T3908 Treat slash-prefixed replacement prompts as provider messages. + +## Phase 40 - Recorded Same-Task Agent CLI Benchmark + +- [x] T4001 Add RED tests for the manifest, complete trial matrix, metrics, and CLI. +- [x] T4002 Require Codex, Claude Code, and Morphic-controlled arms on one task revision. +- [x] T4003 Derive verification and handoff fidelity from predeclared assertions. +- [x] T4004 Reject duplicate, missing, mismatched, and undeclared observations. +- [x] T4005 Report metric-specific leaders without a subjective composite score. +- [x] T4006 Emit deterministic timestamp-free JSON for review and CI artifacts. +- [x] T4007 Keep agent launch and paid live execution outside the offline evaluator. +- [x] T4008 Include the benchmark package in built distributions. + +## Phase 41 - Explicit Opt-in Isolated Trial Recorder + +- [x] T4101 Add RED tests for plan, consent, isolation, cleanup, evidence, and CLI. +- [x] T4102 Require exact arm/check/handoff command coverage for the Phase 40 manifest. +- [x] T4103 Make read-only deterministic planning the default behavior. +- [x] T4104 Require execute, paid acknowledgement, and an explicit estimate cost cap. +- [x] T4105 Run every arm/trial in a unique detached worktree outside the source root. +- [x] T4106 Pass argv without a shell and terminate commands at the configured timeout. +- [x] T4107 Remove worktrees after success, command failure, or recorder exceptions. +- [x] T4108 Persist hashes/byte counts/outcomes without raw prompts or command output. +- [x] T4109 Refuse existing evidence output and write new evidence atomically. +- [x] T4110 Keep actual cost and accepted-patch decisions pending explicit adjudication. + +## Phase 42 - Provider Receipts and Deterministic Adjudication + +- [x] T4201 Add RED tests for receipt parsing, evidence joins, failures, and CLI output. +- [x] T4202 Normalize Codex usage with model-hinted deterministic cost calculation. +- [x] T4203 Normalize Claude provider-reported cost without retaining result text. +- [x] T4204 Define a strict Morphic benchmark receipt envelope. +- [x] T4205 Persist normalized receipts only when parsing succeeds for every trial. +- [x] T4206 Bind independent review decisions to agent argv fingerprints. +- [x] T4207 Recompute machine check/handoff outcomes instead of trusting copied lists. +- [x] T4208 Reject missing/duplicate/mismatched/parse-error/over-cap campaigns. +- [x] T4209 Reject accepted-patch review for provider or process failure. +- [x] T4210 Emit exclusive deterministic Phase 40 result JSON without live execution. + +## Phase 43 - First-party Receipt and Zero-cost Rehearsal + +- [x] T4301 Add RED tests for Morphic receipt output and local rehearsal publication. +- [x] T4302 Add explicit `morphic code --benchmark-receipt` without changing defaults. +- [x] T4303 Aggregate council cost and normalized non-negative usage only. +- [x] T4304 Fail closed without a receipt when failure/cancellation cost is unknown. +- [x] T4305 Commit parseable manifest and recorder configuration examples. +- [x] T4306 Generate internal-only zero-cost fixtures for all three provider shapes. +- [x] T4307 Exercise recorder isolation, receipt parsing, review joins, and finalization. +- [x] T4308 Keep synthetic accepted-patch decisions false and cost exactly zero. +- [x] T4309 Publish a complete rehearsal bundle without replacing existing output. +- [x] T4310 Verify a real pinned-worktree rehearsal leaves no raw output or worktree. + +## Phase 44 - Campaign Preflight and Bound Reviews + +- [x] T4401 Add RED tests for preflight, review templates, bindings, and CLI output. +- [x] T4402 Require the manifest revision to equal a full resolved Git commit. +- [x] T4403 Require exact runtime version declarations matching all three executables. +- [x] T4404 Normalize and fingerprint runtime versions without executing version commands. +- [x] T4405 Reuse deterministic arm/check/handoff command fingerprints. +- [x] T4405a Bind complete manifest/config contracts without exposing the raw goal. +- [x] T4406 Emit a self-fingerprinted preflight with `execution_authorized=false`. +- [x] T4407 Generate null review decisions for the complete expected trial matrix. +- [x] T4408 Bind reviews to preflight, evidence, and expanded agent argv fingerprints. +- [x] T4409 Validate bindings during finalization while accepting legacy unbound reviews. +- [x] T4410 Verify the complete preflight/template path with a zero-cost local campaign. + +## Phase 45 - Reviewer Separation and Campaign Status + +- [x] T4501 Add RED tests for reviewer policy, lifecycle stages, failures, and CLI. +- [x] T4502 Normalize and fingerprint operator/reviewer policy declarations. +- [x] T4503 Reject operator self-review and reviewer IDs outside the allowlist. +- [x] T4504 Enforce minimum distinct reviewers and reject impossible policy capacity. +- [x] T4505 Bind policy SHA-256 into pending and completed review artifacts. +- [x] T4506 Add six deterministic manifest-to-finalized lifecycle stages. +- [x] T4507 Validate artifact order, identity, hashes, estimates, policy, and results. +- [x] T4508 Keep campaign status read-only and non-authorizing at every stage. +- [x] T4509 Preserve legacy review/finalize behavior when no policy binding exists. +- [x] T4510 Verify zero-cost review-pending status without artifact mutation. + +## Phase 46 - Signed Reviewer Provenance + +- [x] T4601 Add RED tests for reviewer trust, signing payloads, signature failures, and CLI paths. +- [x] T4602 Normalize Ed25519 public keys and self-fingerprint the reviewer trust declaration. +- [x] T4603 Bind reviewer trust to the exact benchmark and Phase 45 review policy. +- [x] T4604 Require an active key for each allowed reviewer while retaining revoked keys. +- [x] T4605 Bind completed reviews and each reviewer's decision subset into canonical payloads. +- [x] T4606 Generate signing requests without reading or persisting reviewer private keys. +- [x] T4607 Require one valid active-key signature per distinct reviewer. +- [x] T4608 Reject invalid/revoked/unknown signatures, missing coverage, and mixed artifacts. +- [x] T4609 Add attestation-pending campaign status and trust-bound finalize enforcement. +- [x] T4610 Preserve the unsigned legacy campaign path and verify distribution contents. + +## Phase 47 - Authority-Anchored Campaign Provenance + +- [x] T4701 Add RED tests for authority identity, enrollment, envelopes, failures, and CLI. +- [x] T4702 Normalize and self-fingerprint an offline Ed25519 organization authority. +- [x] T4703 Bind anchored reviewer trust to the exact authority fingerprint. +- [x] T4704 Generate private-key-free enrollment signing requests for every reviewer key. +- [x] T4705 Verify exactly one authority certificate for every retained trust key. +- [x] T4706 Reject invalid signatures, missing coverage, duplicates, and mixed trust artifacts. +- [x] T4707 Require authority enrollment during anchored finalization and status validation. +- [x] T4708 Bind the complete finalized artifact chain into one non-authorizing envelope. +- [x] T4709 Require the authority envelope signature before anchored campaign finalization. +- [x] T4710 Preserve unanchored and unsigned paths and verify distribution contents. + +## Phase 48 - Authority-Root Continuity and Transparency + +- [x] T4801 Add RED tests for root rotation, revocation, Merkle proofs, lifecycle, and CLI. +- [x] T4802 Require contiguous root generations signed by each immediate predecessor. +- [x] T4803 Self-fingerprint and active-root-sign the complete root ledger and revocations. +- [x] T4804 Reject reused roots, unknown revocations, invalid rotations, and revoked active roots. +- [x] T4805 Bind reviewer trust, finalization, and campaign envelopes to the exact root ledger. +- [x] T4806 Build RFC 6962-style domain-separated Merkle roots and inclusion audit paths. +- [x] T4807 Sign tree heads with the ledger's active root and verify exact envelope inclusion. +- [x] T4808 Verify append-only complete-log growth through exact prefix preservation. +- [x] T4809 Add private-key-free rotation/ledger/tree-head and offline log/proof CLI paths. +- [x] T4810 Preserve Phase 47 and unsigned legacy hashes, signatures, and lifecycle behavior. + +## Phase 49 - Compact Consistency and Witness Checkpoints + +- [x] T4901 Add RED tests for compact proofs, witness quorum, split views, lifecycle, and CLI. +- [x] T4902 Generate the unique minimal RFC 6962 consistency path with `SUBPROOF` recursion. +- [x] T4903 Reconstruct and verify both signed roots without complete log artifacts. +- [x] T4904 Reject mismatched log IDs, ledgers, sizes, paths, roots, and tree-head signatures. +- [x] T4905 Normalize and self-fingerprint Ed25519 witness trust with active/revoked keys. +- [x] T4906 Require a strict-majority distinct-witness quorum for checkpoint acceptance. +- [x] T4907 Bind witness signatures to old/new heads, root ledger, proof, and witness trust. +- [x] T4908 Detect same-size different-root witnessed split views. +- [x] T4909 Add offline consistency, witness-trust, and checkpoint-template CLI paths. +- [x] T4910 Preserve Phase 48 inclusion-only and all earlier campaign lifecycle behavior. + +## Phase 50 - Durable Checkpoint Registry and Authenticated Peer Exchange + +- [x] T5001 Add RED tests for registry replay, concurrent append, peer packets, and CLI. +- [x] T5002 Store deterministic self-fingerprinted checkpoint records in hash-chained JSONL. +- [x] T5003 Replay and verify every sequence, record hash, proof, signature, and trust binding. +- [x] T5004 Serialize append with file locking, `O_APPEND`, `fsync`, and mode 0600. +- [x] T5005 Reject stale extensions, sequence gaps, chain tampering, and witnessed split views. +- [x] T5006 Normalize self-fingerprinted peer Ed25519 trust with active/revoked key rotation. +- [x] T5007 Sign exact registry records and authenticate the source peer before import. +- [x] T5008 Make duplicate local append and authenticated packet retry idempotent. +- [x] T5009 Add offline peer-trust, registry status/store/export/import CLI paths. +- [x] T5010 Preserve private-key-free operation and avoid external agents, services, or APIs. + +## Phase 51 - Authenticated Range Sync and Peer Cursors + +- [x] T5101 Add RED tests for signed ranges, overlap/gap handling, acknowledgements, cursors, and CLI. +- [x] T5102 Bind a bounded contiguous record range and its base/head hashes to one peer signature. +- [x] T5103 Authenticate and validate every range record before acquiring the destination write lock. +- [x] T5104 Compare existing overlaps exactly and append only a contiguous missing suffix. +- [x] T5105 Roll back process-level write failures to the original registry size before releasing the lock. +- [x] T5106 Bind receiver acknowledgements to the exact signed range and applied registry head. +- [x] T5107 Store verified acknowledgements in a mode 0600 hash-chained peer cursor ledger. +- [x] T5108 Enforce monotonic per-peer cursors and reject regression, conflict, tampering, and invalid signatures. +- [x] T5109 Add private-key-free range export/import, acknowledgement-template, and cursor CLI paths. +- [x] T5110 Preserve single-record exchange and keep all network/paid execution disabled. + +## Phase 52 - Peer-Trust Rollover Continuity + +- [x] T5201 Add RED tests for rollover quorum, generation replay, resolver, historical cursors, and CLI. +- [x] T5202 Bind registry, generation, predecessor/successor trust, and computed quorum in one statement. +- [x] T5203 Emit one private-key-free signing request per active predecessor peer. +- [x] T5204 Require a strict majority of distinct predecessor peers with active trusted Ed25519 keys. +- [x] T5205 Reject minority, duplicate peers, successor-only/revoked keys, invalid signatures, and trust reuse. +- [x] T5206 Build a contiguous self-fingerprinted generation ledger from out-of-band genesis. +- [x] T5207 Resolve historical trust fingerprints so old and new acknowledgements replay together. +- [x] T5208 Extend cursor store/status to accept exactly one trust snapshot or trust ledger. +- [x] T5209 Add peer-trust rotation-template and generation-ledger CLI paths. +- [x] T5210 Preserve offline/private-key-free behavior and defer network transport. + +## Phase 53 - Authenticated Loopback Checkpoint Gossip + +- [x] T5301 Add RED tests for challenge authentication, replay, bounds, timeout, shutdown, artifacts, and CLI. +- [x] T5302 Bind an explicit listener only to `127.0.0.1` and publish a protocol-versioned mode 0600 descriptor. +- [x] T5303 Authenticate each request and response with one-use client/server nonces and HMAC-SHA256. +- [x] T5304 Bind protocol, instance, registry, source peer, operation, and payload into transport authentication. +- [x] T5305 Bound request/response bytes, concurrency, retained nonces, request count, timeout, and listener lifetime. +- [x] T5306 Serve only pre-signed exact checkpoint ranges and verify peer signatures before listening. +- [x] T5307 Reverify fetched range signatures under a trust snapshot or generation ledger on the client. +- [x] T5308 Verify submitted acknowledgements and append them through the durable monotonic cursor store. +- [x] T5309 Close the listener and all active clients deterministically and remove only the owned descriptor. +- [x] T5310 Add explicit opt-in serve/status/fetch/ack CLI paths while deferring remote TLS and automatic signing. + +## Phase 54 - Durable Bounded Gossip Catch-Up Loop + +- [x] T5401 Add RED tests for multi-range catch-up, resume, retry, crash recovery, budgets, rollback, tampering, and CLI. +- [x] T5402 Add a mode 0600 append-only sync audit with one non-blocking whole-loop process lock. +- [x] T5403 Self-fingerprint and hash-chain imported, retry, recovered, trust-advanced, and stopped audit records. +- [x] T5404 Pin the highest verified peer-trust generation and reject older or pinned-generation fork ledgers before network access. +- [x] T5405 Reconcile a verified registry ahead of its audit after a crash without re-importing existing records. +- [x] T5406 Pull a signed range containing the exact next local sequence and atomically import only its missing suffix. +- [x] T5407 Bound rounds, newly imported records, attempts, and deterministic retry delays. +- [x] T5408 Return explicit up-to-date, gap, record-budget, round-budget, and retry-exhausted stop reasons. +- [x] T5409 Keep secrets and raw exception text out of deterministic audit records. +- [x] T5410 Add a private-key-free gossip-sync CLI while deferring acknowledgement signing and remote mTLS. + +## Phase 55 - Peer-Signed Remote Mutual TLS + +- [x] T5501 Add RED tests for enrollment signatures, certificate rotation, mTLS exchange, replay, hostname/address rejection, plaintext rejection, cleanup, and CLI. +- [x] T5502 Bind exact leaf DER and SPKI SHA-256 pins, subject/issuer, serial, validity, SANs, and client/server EKUs into deterministic enrollment statements. +- [x] T5503 Require one active peer Ed25519 identity signature and reverify it against the exact peer-trust fingerprint. +- [x] T5504 Chain monotonic per-peer certificate generations and reject missing, mismatched, duplicate, or cross-peer active pins. +- [x] T5505 Publish self-fingerprinted TLS trust with exactly one active enrollment per included peer and reject inactive certificate use. +- [x] T5506 Add TLS 1.3-only mutual certificate authentication with no token or plaintext fallback. +- [x] T5507 Require certificate hostname verification plus explicit canonical client/server IP allowlists and peer-id-to-leaf-pin matching. +- [x] T5508 Preserve bounded request/response size, concurrency, nonce retention, timeout, request capacity, and deterministic descriptor/client cleanup. +- [x] T5509 Add private-key-free enrollment-template, enrollment-finalization, trust, mTLS-serve, and mTLS-status CLI paths. +- [x] T5510 Document CA/out-of-band trust boundaries and defer mTLS fetch/ack/sync CLI wiring, discovery, automatic signing, and revocation status. + +## Phase 56 - Mutual-TLS Artifact Sync + +- [x] T5601 Add RED tests for transport-neutral status/fetch/ack operations, sync sender injection, reusable mTLS requests, and CLI registration. +- [x] T5602 Define one typed authenticated request-sender protocol while preserving protocol-v1 loopback as the default. +- [x] T5603 Route status, signed range fetch, and signed acknowledgement submission through the injected sender. +- [x] T5604 Keep peer-signature and returned-artifact verification after transport completion on every sender. +- [x] T5605 Route the existing bounded resumable sync loop through the same sender without changing audit, retry, recovery, budget, or rollback semantics. +- [x] T5606 Add a reusable TLS 1.3 mutual-auth client that retains exact certificate, hostname, address, and TLS-trust pins. +- [x] T5607 Add mTLS fetch and acknowledgement CLI paths with exactly one artifact peer-trust snapshot or generation ledger. +- [x] T5608 Add an mTLS sync CLI backed by the active generation's reverified TLS enrollment trust. +- [x] T5609 Preserve the token-free descriptor, no-plaintext-fallback policy, and all existing protocol bounds. +- [x] T5610 Document the remaining certificate revocation/expiry policy, discovery, and automatic signing boundaries. + ## Deferred - [ ] D001 Textual full-screen TUI. -- [ ] D002 Claude Code adapter. +- [x] D002 Claude Code adapter. - [ ] D003 Gemini CLI adapter. -- [ ] D004 Codex CLI adapter. +- [x] D004 Codex CLI adapter. - [ ] D005 OpenHands adapter. - [ ] D006 `.morphic` to `.claude` export. - [ ] D007 `.morphic` to Gemini/Codex metadata export. -- [ ] D008 Hook command execution. +- [x] D008 Hook command execution. - [ ] D009 Council event visualization. - [ ] D010 Memory candidate approval UI. +- [ ] D011 General chat tool execution UX beyond explicit hook commands. +- [ ] D012 Native CLI streaming, resume, steering, and permission propagation. diff --git a/tests/unit/application/test_chat_hook_execution.py b/tests/unit/application/test_chat_hook_execution.py index 64e5fa2..764cd87 100644 --- a/tests/unit/application/test_chat_hook_execution.py +++ b/tests/unit/application/test_chat_hook_execution.py @@ -7,8 +7,15 @@ from application.use_cases.plan_chat_hooks import PlanChatHooksUseCase from domain.entities.chat_event import ChatEvent, ChatEventType from domain.entities.chat_session import ChatSession, PermissionMode -from domain.entities.hook import HookDefinition, HookDiagnostic, HookType +from domain.entities.hook import ( + HookDefinition, + HookDiagnostic, + HookExecutionRequest, + HookExecutionResult, + HookType, +) from domain.ports.chat_session_store import ChatSessionStorePort +from domain.ports.hook_executor import HookExecutorPort from domain.ports.hook_registry import HookRegistryPort pytestmark = pytest.mark.asyncio @@ -45,6 +52,21 @@ def hooks_for(self, hook_type: HookType) -> list[HookDefinition]: return [hook for hook in self._hooks if hook.hook_type is hook_type] +class FakeHookExecutor(HookExecutorPort): + def __init__(self) -> None: + self.requests: list[HookExecutionRequest] = [] + + async def execute(self, request: HookExecutionRequest) -> HookExecutionResult: + self.requests.append(request) + return HookExecutionResult( + request_id=request.id, + success=True, + stdout_summary="hook ok", + stderr_summary="", + exit_code=0, + ) + + def _session() -> ChatSession: return ChatSession.start( session_id="chat-1", @@ -147,3 +169,125 @@ async def test_plan_chat_hooks_blocks_when_hook_diagnostics_fail() -> None: assert "Hook diagnostics failed" in str(exc_info.value) assert store.appended == [] + + +async def test_execute_chat_hook_runs_enabled_hook_and_records_result_events() -> None: + from application.use_cases.execute_chat_hook import ExecuteChatHookUseCase + + store = InMemoryChatSessionStore() + executor = FakeHookExecutor() + registry = FakeHookRegistry( + diagnostics=[ + HookDiagnostic( + name="Hook: echo", + status="OK", + message="pre_tool hook is valid", + ) + ], + hooks=[ + HookDefinition( + name="echo", + hook_type=HookType.PRE_TOOL, + command="echo before", + enabled=True, + source_path=".morphic/hooks/echo.json", + ) + ], + ) + + result = await ExecuteChatHookUseCase( + session_store=store, + hook_registry=registry, + hook_executor=executor, + ).execute(session=_session(), hook_type=HookType.PRE_TOOL) + + assert len(executor.requests) == 1 + assert executor.requests[0].hook_name == "echo" + assert executor.requests[0].command == "echo before" + assert [event.type for event in result.events] == [ + ChatEventType.HOOK_EXECUTION_REQUESTED, + ChatEventType.HOOK_EXECUTION_COMPLETED, + ] + assert result.events[0].payload["hook_name"] == "echo" + assert result.events[0].payload["command"] == "echo before" + assert result.events[1].payload["success"] is True + assert result.events[1].payload["stdout_summary"] == "hook ok" + assert store.appended == result.events + assert result.session.next_sequence == 2 + + +async def test_execute_chat_hook_skips_disabled_hooks_without_executor_call() -> None: + from application.use_cases.execute_chat_hook import ExecuteChatHookUseCase + + store = InMemoryChatSessionStore() + executor = FakeHookExecutor() + registry = FakeHookRegistry( + diagnostics=[ + HookDiagnostic( + name="Hook: disabled", + status="WARN", + message="post_tool hook is disabled", + ) + ], + hooks=[ + HookDefinition( + name="disabled", + hook_type=HookType.POST_TOOL, + command="echo skipped", + enabled=False, + source_path=".morphic/hooks/disabled.json", + ) + ], + ) + + result = await ExecuteChatHookUseCase( + session_store=store, + hook_registry=registry, + hook_executor=executor, + ).execute(session=_session(), hook_type=HookType.POST_TOOL) + + assert executor.requests == [] + assert [event.type for event in result.events] == [ + ChatEventType.HOOK_EXECUTION_SKIPPED + ] + assert result.events[0].payload["hook_name"] == "disabled" + assert result.events[0].payload["status"] == "skipped" + assert result.events[0].payload["reason"] == "disabled" + assert store.appended == result.events + assert result.session.next_sequence == 1 + + +async def test_execute_chat_hook_blocks_when_hook_diagnostics_fail() -> None: + from application.use_cases.execute_chat_hook import ExecuteChatHookUseCase + + store = InMemoryChatSessionStore() + executor = FakeHookExecutor() + registry = FakeHookRegistry( + diagnostics=[ + HookDiagnostic( + name="Hook: secret", + status="FAIL", + message="command references a secret path", + ) + ], + hooks=[ + HookDefinition( + name="secret", + hook_type=HookType.PRE_SHELL, + command="cat .env", + enabled=True, + source_path=".morphic/hooks/secret.json", + ) + ], + ) + + with pytest.raises(ValueError) as exc_info: + await ExecuteChatHookUseCase( + session_store=store, + hook_registry=registry, + hook_executor=executor, + ).execute(session=_session(), hook_type=HookType.PRE_SHELL) + + assert "Hook diagnostics failed" in str(exc_info.value) + assert executor.requests == [] + assert store.appended == [] diff --git a/tests/unit/application/test_chat_session_use_cases.py b/tests/unit/application/test_chat_session_use_cases.py index 72a901f..922d45e 100644 --- a/tests/unit/application/test_chat_session_use_cases.py +++ b/tests/unit/application/test_chat_session_use_cases.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio + import pytest from application.use_cases.discover_workspace_context import ( @@ -13,6 +15,7 @@ from application.use_cases.send_chat_message import SendChatMessageUseCase from application.use_cases.start_chat_session import StartChatSessionUseCase from application.use_cases.summarize_chat_session import SummarizeChatSessionUseCase +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType from domain.entities.chat_event import ChatEvent, ChatEventType from domain.entities.chat_session import ChatSessionStatus, PermissionMode from domain.entities.council_runtime import CouncilDecision, CouncilRole, CouncilTurn @@ -21,11 +24,13 @@ ContextSourceType, WorkspaceContextSource, ) +from domain.ports.agent_engine import AgentEngineEventSinkPort from domain.ports.chat_session_store import ChatSessionStorePort from domain.ports.context_discovery import ContextDiscoveryPort -from domain.ports.council_runtime import CouncilRuntimePort +from domain.ports.council_runtime import CouncilRuntimePort, StreamingCouncilRuntimePort from domain.ports.engine_registry import EngineProfile, EngineRegistryPort, EngineRuntimeKind from domain.value_objects import RiskLevel +from domain.value_objects.agent_engine import AgentEngineType class InMemoryChatSessionStore(ChatSessionStorePort): @@ -89,6 +94,17 @@ async def deliberate( ) +class CancellingCouncilRuntime(CouncilRuntimePort): + async def deliberate( + self, + session, + context: ContextIndex, + user_message: str, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + del session, context, user_message + raise asyncio.CancelledError + + class FakeEngineRegistry(EngineRegistryPort): async def list_engines(self) -> list[EngineProfile]: return [ @@ -106,6 +122,114 @@ async def get_engine(self, engine_id: str) -> EngineProfile | None: return next((engine for engine in engines if engine.id == engine_id), None) +class FakeNativeEventRuntime(CouncilRuntimePort): + async def deliberate( + self, + session, + context: ContextIndex, + user_message: str, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + engine_event = AgentEngineEvent( + type=AgentEngineEventType.TOOL_COMPLETED, + engine=AgentEngineType.CODEX_CLI, + sequence=3, + session_id="thread-1", + item_id="item-1", + item_type="command_execution", + text="pytest -q", + payload={"exit_code": 0}, + ) + turn = CouncilTurn( + role=CouncilRole.IMPLEMENTER, + engine_id="codex_cli", + content="Tests pass.", + engine_events=[engine_event], + ) + return ( + [turn], + CouncilDecision( + leader_engine_id="codex_cli", + selected_role=CouncilRole.IMPLEMENTER, + selected_content=turn.content, + rationale="Native engine completed the task.", + ), + ) + + +class FakeStreamingNativeEventRuntime(StreamingCouncilRuntimePort): + def __init__(self, store: InMemoryChatSessionStore) -> None: + self._store = store + + async def deliberate(self, session, context, user_message): + raise AssertionError("streaming path expected") + + async def deliberate_stream( + self, + session, + context, + user_message, + event_sink: AgentEngineEventSinkPort, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + del session, context, user_message + engine_event = AgentEngineEvent( + type=AgentEngineEventType.PROGRESS, + engine=AgentEngineType.CODEX_CLI, + sequence=0, + session_id="thread-live", + text="working", + payload={"type": "item.updated"}, + ) + await event_sink.publish(engine_event) + assert [event.type for event in self._store.appended[-2:]] == [ + ChatEventType.USER_MESSAGE, + ChatEventType.ENGINE_EVENT, + ] + turn = CouncilTurn( + role=CouncilRole.IMPLEMENTER, + engine_id="codex_cli", + content="Done.", + engine_events=[engine_event], + ) + return [turn], CouncilDecision( + leader_engine_id="codex_cli", + selected_role=CouncilRole.IMPLEMENTER, + selected_content=turn.content, + rationale="Native stream completed.", + ) + + +class CancellingStreamingNativeEventRuntime(StreamingCouncilRuntimePort): + async def deliberate(self, session, context, user_message): + raise AssertionError("streaming path expected") + + async def deliberate_stream( + self, + session, + context, + user_message, + event_sink: AgentEngineEventSinkPort, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + del session, context, user_message + await event_sink.publish( + AgentEngineEvent( + type=AgentEngineEventType.RUN_STARTED, + engine=AgentEngineType.CODEX_CLI, + sequence=0, + session_id="thread-cancelled", + payload={"type": "thread.started"}, + ) + ) + raise asyncio.CancelledError + + +class CollectingEngineEventObserver(AgentEngineEventSinkPort): + def __init__(self) -> None: + self.events: list[AgentEngineEvent] = [] + + async def publish(self, event: AgentEngineEvent) -> None: + self.events.append(event) + + @pytest.mark.asyncio async def test_start_chat_session_appends_session_started_event() -> None: store = InMemoryChatSessionStore() @@ -149,6 +273,41 @@ async def test_resume_latest_rebuilds_session_sequence_from_events() -> None: assert result.events[0] == started.event +@pytest.mark.asyncio +async def test_resume_restores_native_session_provenance() -> None: + store = InMemoryChatSessionStore() + await StartChatSessionUseCase(session_store=store).execute( + goal="Fix tests", + permission_mode=PermissionMode.WORKSPACE_WRITE, + session_id="chat-native-resume", + ) + await store.append_event( + ChatEvent( + type=ChatEventType.CONTEXT_INDEXED, + session_id="chat-native-resume", + sequence=1, + payload={"workspace_root": "/repo"}, + ) + ) + await store.append_event( + ChatEvent( + type=ChatEventType.ENGINE_EVENT, + session_id="chat-native-resume", + sequence=2, + payload={"engine": "codex_cli", "session_id": "thread-1"}, + ) + ) + + result = await ResumeChatSessionUseCase(session_store=store).execute( + "chat-native-resume" + ) + + native = result.session.native_sessions["codex_cli"] + assert native.session_id == "thread-1" + assert native.workspace_root == "/repo" + assert native.permission_mode is PermissionMode.WORKSPACE_WRITE + + @pytest.mark.asyncio async def test_send_chat_message_appends_user_council_and_assistant_events() -> None: store = InMemoryChatSessionStore() @@ -181,6 +340,120 @@ async def test_send_chat_message_appends_user_council_and_assistant_events() -> assert result.events[-1].payload["text"] == "Start with application use cases." +@pytest.mark.asyncio +async def test_send_chat_message_records_cancelled_non_streaming_turn() -> None: + store = InMemoryChatSessionStore() + session = ( + await StartChatSessionUseCase(session_store=store).execute( + goal="Plan changes", + permission_mode=PermissionMode.READ_ONLY, + session_id="chat-local-cancelled-1", + ) + ).session + context = await FakeContextDiscovery().discover("/repo") + + with pytest.raises(asyncio.CancelledError): + await SendChatMessageUseCase( + session_store=store, + council_runtime=CancellingCouncilRuntime(), + ).execute(session=session, context=context, message="plan changes") + + assert [event.type for event in store.appended[-2:]] == [ + ChatEventType.USER_MESSAGE, + ChatEventType.TURN_CANCELLED, + ] + + +@pytest.mark.asyncio +async def test_send_chat_message_persists_native_engine_events_before_turn() -> None: + store = InMemoryChatSessionStore() + session = ( + await StartChatSessionUseCase(session_store=store).execute( + goal="Fix tests", + permission_mode=PermissionMode.WORKSPACE_WRITE, + session_id="chat-native-1", + ) + ).session + context = await FakeContextDiscovery().discover("/repo") + + result = await SendChatMessageUseCase( + session_store=store, + council_runtime=FakeNativeEventRuntime(), + ).execute( + session=session, + context=context, + message="fix tests", + ) + + assert [event.type for event in result.events] == [ + ChatEventType.USER_MESSAGE, + ChatEventType.ENGINE_EVENT, + ChatEventType.COUNCIL_ARGUMENT, + ChatEventType.COUNCIL_DECISION, + ChatEventType.ASSISTANT_MESSAGE, + ] + native = result.events[1] + assert native.payload["engine"] == "codex_cli" + assert native.payload["type"] == "tool_completed" + assert native.payload["payload"]["exit_code"] == 0 + + +@pytest.mark.asyncio +async def test_send_chat_message_persists_stream_events_live_without_duplicates() -> None: + store = InMemoryChatSessionStore() + session = ( + await StartChatSessionUseCase(session_store=store).execute( + goal="Fix tests", + permission_mode=PermissionMode.WORKSPACE_WRITE, + session_id="chat-stream-1", + ) + ).session + context = await FakeContextDiscovery().discover("/repo") + + observer = CollectingEngineEventObserver() + result = await SendChatMessageUseCase( + session_store=store, + council_runtime=FakeStreamingNativeEventRuntime(store), + engine_event_observer=observer, + ).execute(session=session, context=context, message="fix tests") + + assert [event.type for event in result.events] == [ + ChatEventType.USER_MESSAGE, + ChatEventType.ENGINE_EVENT, + ChatEventType.COUNCIL_ARGUMENT, + ChatEventType.COUNCIL_DECISION, + ChatEventType.ASSISTANT_MESSAGE, + ] + assert sum(event.type is ChatEventType.ENGINE_EVENT for event in store.appended) == 1 + assert [event.type for event in observer.events] == [AgentEngineEventType.PROGRESS] + + +@pytest.mark.asyncio +async def test_send_chat_message_records_cancelled_stream_turn_and_propagates() -> None: + store = InMemoryChatSessionStore() + session = ( + await StartChatSessionUseCase(session_store=store).execute( + goal="Fix tests", + permission_mode=PermissionMode.WORKSPACE_WRITE, + session_id="chat-cancelled-1", + ) + ).session + context = await FakeContextDiscovery().discover("/repo") + + with pytest.raises(asyncio.CancelledError): + await SendChatMessageUseCase( + session_store=store, + council_runtime=CancellingStreamingNativeEventRuntime(), + ).execute(session=session, context=context, message="fix tests") + + assert [event.type for event in store.appended[-3:]] == [ + ChatEventType.USER_MESSAGE, + ChatEventType.ENGINE_EVENT, + ChatEventType.TURN_CANCELLED, + ] + assert store.appended[-1].payload == {"reason": "caller_cancelled"} + + @pytest.mark.asyncio async def test_discover_workspace_context_appends_context_indexed_event() -> None: store = InMemoryChatSessionStore() diff --git a/tests/unit/application/test_chat_tool_execution.py b/tests/unit/application/test_chat_tool_execution.py index 2435245..3f007aa 100644 --- a/tests/unit/application/test_chat_tool_execution.py +++ b/tests/unit/application/test_chat_tool_execution.py @@ -4,12 +4,20 @@ import pytest +from application.use_cases.execute_chat_hook import ExecuteChatHookUseCase from application.use_cases.execute_chat_tool import ExecuteChatToolUseCase from application.use_cases.plan_chat_hooks import PlanChatHooksUseCase from domain.entities.chat_event import ChatEvent, ChatEventType from domain.entities.chat_session import ChatSession, PermissionMode -from domain.entities.hook import HookDefinition, HookDiagnostic, HookType +from domain.entities.hook import ( + HookDefinition, + HookDiagnostic, + HookExecutionRequest, + HookExecutionResult, + HookType, +) from domain.ports.chat_session_store import ChatSessionStorePort +from domain.ports.hook_executor import HookExecutorPort from domain.ports.hook_registry import HookRegistryPort from domain.ports.tool_executor import ( ToolExecutionRequest, @@ -49,6 +57,22 @@ async def execute(self, request: ToolExecutionRequest) -> ToolExecutionResult: ) +class FakeHookExecutor(HookExecutorPort): + def __init__(self, *, success: bool = True) -> None: + self.requests: list[HookExecutionRequest] = [] + self._success = success + + async def execute(self, request: HookExecutionRequest) -> HookExecutionResult: + self.requests.append(request) + return HookExecutionResult( + request_id=request.id, + success=self._success, + stdout_summary=f"ran {request.hook_name}" if self._success else "", + stderr_summary="" if self._success else f"failed {request.hook_name}", + exit_code=0 if self._success else 1, + ) + + class FakeHookRegistry(HookRegistryPort): def __init__(self, hooks: list[HookDefinition]) -> None: self._hooks = hooks @@ -183,3 +207,120 @@ async def test_execute_tool_records_pre_and_post_hook_plan_events() -> None: assert [event.sequence for event in store.appended] == list(range(6)) assert store.appended == result.events assert result.session.next_sequence == 6 + + +@pytest.mark.asyncio +async def test_execute_tool_runs_pre_and_post_hooks_when_hook_runner_is_injected() -> None: + store = InMemoryChatSessionStore() + executor = FakeToolExecutor() + hook_executor = FakeHookExecutor() + hook_runner = ExecuteChatHookUseCase( + session_store=store, + hook_registry=FakeHookRegistry( + [ + HookDefinition( + name="pre-log", + hook_type=HookType.PRE_TOOL, + command="echo before", + enabled=True, + source_path=".morphic/hooks/pre-log.json", + ), + HookDefinition( + name="post-log", + hook_type=HookType.POST_TOOL, + command="echo after", + enabled=True, + source_path=".morphic/hooks/post-log.json", + ), + ] + ), + hook_executor=hook_executor, + ) + session = ChatSession.start( + session_id="chat-1", + goal="write file", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + + result = await ExecuteChatToolUseCase( + session_store=store, + tool_executor=executor, + hook_runner=hook_runner, + ).execute( + session=session, + tool_name="fs_write", + arguments={"path": "x.txt", "content": "x"}, + risk_level=RiskLevel.MEDIUM, + diff_summary="create x.txt", + verification_label="unit tests", + ) + + assert [request.hook_name for request in hook_executor.requests] == [ + "pre-log", + "post-log", + ] + assert [event.type for event in result.events] == [ + ChatEventType.HOOK_EXECUTION_REQUESTED, + ChatEventType.HOOK_EXECUTION_COMPLETED, + ChatEventType.DIFF_PROPOSED, + ChatEventType.TOOL_CALL_REQUESTED, + ChatEventType.TOOL_CALL_COMPLETED, + ChatEventType.VERIFICATION_RESULT, + ChatEventType.HOOK_EXECUTION_REQUESTED, + ChatEventType.HOOK_EXECUTION_COMPLETED, + ] + assert result.events[0].payload["hook_name"] == "pre-log" + assert result.events[1].payload["stdout_summary"] == "ran pre-log" + assert result.events[-2].payload["hook_name"] == "post-log" + assert result.events[-1].payload["stdout_summary"] == "ran post-log" + assert [event.sequence for event in store.appended] == list(range(8)) + assert store.appended == result.events + assert result.session.next_sequence == 8 + + +@pytest.mark.asyncio +async def test_execute_tool_stops_when_pre_hook_execution_fails() -> None: + store = InMemoryChatSessionStore() + executor = FakeToolExecutor() + hook_executor = FakeHookExecutor(success=False) + hook_runner = ExecuteChatHookUseCase( + session_store=store, + hook_registry=FakeHookRegistry( + [ + HookDefinition( + name="pre-log", + hook_type=HookType.PRE_TOOL, + command="false", + enabled=True, + source_path=".morphic/hooks/pre-log.json", + ) + ] + ), + hook_executor=hook_executor, + ) + session = ChatSession.start( + session_id="chat-1", + goal="write file", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + + with pytest.raises(RuntimeError) as exc_info: + await ExecuteChatToolUseCase( + session_store=store, + tool_executor=executor, + hook_runner=hook_runner, + ).execute( + session=session, + tool_name="fs_write", + arguments={"path": "x.txt", "content": "x"}, + risk_level=RiskLevel.MEDIUM, + diff_summary="create x.txt", + ) + + assert "pre_tool hook failed" in str(exc_info.value) + assert executor.requests == [] + assert [event.type for event in store.appended] == [ + ChatEventType.HOOK_EXECUTION_REQUESTED, + ChatEventType.HOOK_EXECUTION_COMPLETED, + ] + assert store.appended[-1].payload["success"] is False diff --git a/tests/unit/application/test_route_to_engine.py b/tests/unit/application/test_route_to_engine.py index 7645c29..5cc055c 100644 --- a/tests/unit/application/test_route_to_engine.py +++ b/tests/unit/application/test_route_to_engine.py @@ -11,8 +11,15 @@ import pytest from application.use_cases.route_to_engine import RouteToEngineUseCase +from domain.entities.chat_session import PermissionMode from domain.entities.cognitive import AgentAffinityScore, SharedTaskState -from domain.ports.agent_engine import AgentEngineCapabilities, AgentEnginePort, AgentEngineResult +from domain.ports.agent_engine import ( + AgentEngineCapabilities, + AgentEngineEventSinkPort, + AgentEnginePort, + AgentEngineResult, + ResumableStreamingScopedAgentEnginePort, +) from domain.value_objects.agent_engine import AgentEngineType from domain.value_objects.model_tier import TaskType @@ -28,22 +35,31 @@ def _make_driver( cost_per_hour_usd: float = 0.0, ) -> AsyncMock: """Create a mock AgentEnginePort driver.""" - driver = AsyncMock(spec=AgentEnginePort) + scoped_engines = {AgentEngineType.CODEX_CLI, AgentEngineType.CLAUDE_CODE} + driver_spec = ( + ResumableStreamingScopedAgentEnginePort + if engine_type in scoped_engines + else AgentEnginePort + ) + driver = AsyncMock(spec=driver_spec) driver.is_available = AsyncMock(return_value=available) driver.get_capabilities.return_value = AgentEngineCapabilities( engine_type=engine_type, max_context_tokens=max_context_tokens, cost_per_hour_usd=cost_per_hour_usd, ) - driver.run_task = AsyncMock( - return_value=AgentEngineResult( - engine=engine_type, - success=success, - output=output, - cost_usd=cost_usd, - error=error, - ) + result = AgentEngineResult( + engine=engine_type, + success=success, + output=output, + cost_usd=cost_usd, + error=error, ) + driver.run_task = AsyncMock(return_value=result) + if engine_type in scoped_engines: + driver.run_task_scoped = AsyncMock(return_value=result) + driver.run_task_scoped_stream = AsyncMock(return_value=result) + driver.resume_task_scoped_stream = AsyncMock(return_value=result) return driver @@ -223,6 +239,140 @@ async def test_passes_model_and_timeout(self, drivers: dict) -> None: timeout_seconds=60.0, ) + async def test_passes_workspace_and_permission_context_to_codex(self, drivers: dict) -> None: + uc = RouteToEngineUseCase(drivers) + await uc.execute( + "fix tests", + preferred_engine=AgentEngineType.CODEX_CLI, + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + + drivers[AgentEngineType.CODEX_CLI].run_task_scoped.assert_awaited_once_with( + task="fix tests", + model=None, + timeout_seconds=300.0, + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + + async def test_scoped_execution_skips_unsupported_engine(self) -> None: + openhands = _make_driver(AgentEngineType.OPENHANDS) + codex = _make_driver(AgentEngineType.CODEX_CLI) + uc = RouteToEngineUseCase( + { + AgentEngineType.OPENHANDS: openhands, + AgentEngineType.CODEX_CLI: codex, + } + ) + + result = await uc.execute( + "fix tests", + task_type=TaskType.CODE_GENERATION, + budget=5.0, + preferred_engine=AgentEngineType.OPENHANDS, + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + + assert result.success is True + assert result.engine is AgentEngineType.CODEX_CLI + openhands.run_task.assert_not_awaited() + codex.run_task_scoped.assert_awaited_once() + assert result.fallback_attempts[0].skip_reason == "scoped_execution_unsupported" + + async def test_streaming_scoped_execution_uses_streaming_driver_method( + self, + drivers: dict, + ) -> None: + sink = AsyncMock(spec=AgentEngineEventSinkPort) + uc = RouteToEngineUseCase(drivers) + + result = await uc.execute( + "fix tests", + preferred_engine=AgentEngineType.CODEX_CLI, + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + ) + + assert result.success is True + drivers[AgentEngineType.CODEX_CLI].run_task_scoped_stream.assert_awaited_once_with( + task="fix tests", + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + model=None, + timeout_seconds=300.0, + ) + drivers[AgentEngineType.CODEX_CLI].run_task_scoped.assert_not_awaited() + + async def test_native_resume_uses_resumable_streaming_driver_method( + self, + drivers: dict, + ) -> None: + sink = AsyncMock(spec=AgentEngineEventSinkPort) + uc = RouteToEngineUseCase(drivers) + + result = await uc.execute( + "continue", + preferred_engine=AgentEngineType.CODEX_CLI, + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + resume_session_id="thread-1", + resume_engine=AgentEngineType.CODEX_CLI, + ) + + assert result.success is True + drivers[ + AgentEngineType.CODEX_CLI + ].resume_task_scoped_stream.assert_awaited_once_with( + task="continue", + resume_session_id="thread-1", + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + model=None, + timeout_seconds=300.0, + ) + drivers[AgentEngineType.CODEX_CLI].run_task_scoped_stream.assert_not_awaited() + + async def test_native_resume_never_falls_back_to_another_engine(self) -> None: + claude = _make_driver( + AgentEngineType.CLAUDE_CODE, + success=False, + error="session missing", + ) + codex = _make_driver(AgentEngineType.CODEX_CLI) + sink = AsyncMock(spec=AgentEngineEventSinkPort) + uc = RouteToEngineUseCase( + { + AgentEngineType.CLAUDE_CODE: claude, + AgentEngineType.CODEX_CLI: codex, + } + ) + + result = await uc.execute( + "continue", + preferred_engine=AgentEngineType.CLAUDE_CODE, + task_type=TaskType.CODE_GENERATION, + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + resume_session_id="claude-session-1", + resume_engine=AgentEngineType.CLAUDE_CODE, + ) + + assert result.success is False + claude.resume_task_scoped_stream.assert_awaited_once() + codex.resume_task_scoped_stream.assert_not_awaited() + assert any( + attempt.engine == "codex_cli" + and attempt.skip_reason == "resume_engine_mismatch" + for attempt in result.fallback_attempts + ) + # ═══════════════════════════════════════════════════════════════ # execute — fallback diff --git a/tests/unit/benchmarks/test_agent_cli_adjudication.py b/tests/unit/benchmarks/test_agent_cli_adjudication.py new file mode 100644 index 0000000..88f6be8 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_adjudication.py @@ -0,0 +1,375 @@ +"""Tests for provider receipts and deterministic benchmark adjudication.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from benchmarks.agent_cli_adjudication import ( + AdjudicationReviews, + RecordedEvidence, + finalize_recorded_results, + finalized_results_json, +) +from benchmarks.agent_cli_comparison import AgentCliManifest +from benchmarks.agent_cli_receipts import ProviderReceipt, ProviderReceiptParser +from interface.cli.main import app + +runner = CliRunner() + + +def _manifest() -> dict[str, object]: + return { + "schema_version": 1, + "benchmark_id": "adjudicate-001", + "task": { + "id": "task-001", + "goal": "Implement the same change", + "workspace_revision": "abc123", + "checks": ["unit"], + "handoff_assertions": ["decision"], + }, + "arms": ["codex_cli", "claude_code", "morphic_control"], + "repetitions": 1, + } + + +def _command(*, exit_code: int = 0) -> dict[str, object]: + return { + "argv_sha256": "a" * 64, + "exit_code": exit_code, + "timed_out": False, + "elapsed_seconds": 10.0, + "stdout_sha256": "b" * 64, + "stdout_bytes": 100, + "stderr_sha256": "c" * 64, + "stderr_bytes": 0, + } + + +def _receipt(arm: str, cost: float) -> dict[str, object]: + sources = { + "codex_cli": "calculated_from_usage", + "claude_code": "provider_reported", + "morphic_control": "morphic_reported", + } + return { + "provider": arm, + "success": True, + "model": "o4-mini" if arm == "codex_cli" else f"{arm}-model", + "usage": {"input_tokens": 100, "output_tokens": 20}, + "cost_usd": cost, + "cost_source": sources[arm], + "parse_errors": 0, + } + + +def _evidence() -> dict[str, object]: + trials = [] + for arm, cost in ( + ("codex_cli", 0.000198), + ("claude_code", 0.2), + ("morphic_control", 0.3), + ): + trials.append( + { + "arm": arm, + "trial": 1, + "reserved_cost_usd": cost, + "agent": _command(), + "checks": {"unit": _command()}, + "handoff_assertions": {"decision": _command()}, + "receipt": _receipt(arm, cost), + "completed": True, + "passed_checks": ["unit"], + "passed_handoff_assertions": ["decision"], + } + ) + return { + "schema_version": 1, + "benchmark_id": "adjudicate-001", + "task_id": "task-001", + "workspace_revision": "abc123", + "estimated_max_cost_usd": 0.6, + "authorized_cost_cap_usd": 1.0, + "cost_collection": "normalized_receipts", + "trials": trials, + } + + +def _reviews() -> dict[str, object]: + return { + "schema_version": 1, + "benchmark_id": "adjudicate-001", + "task_id": "task-001", + "workspace_revision": "abc123", + "decisions": [ + { + "arm": arm, + "trial": 1, + "agent_argv_sha256": "a" * 64, + "accepted_patch": True, + "human_interventions": index, + "recovery_attempted": index > 0, + "recovery_succeeded": index > 0, + "reviewer_id": "reviewer-1", + "review_artifact_sha256": "d" * 64, + } + for index, arm in enumerate( + ("codex_cli", "claude_code", "morphic_control") + ) + ], + } + + +def test_parse_codex_receipt_calculates_cost_from_usage() -> None: + stdout = "\n".join( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-1"}), + json.dumps( + { + "type": "turn.completed", + "usage": {"input_tokens": 1000, "output_tokens": 100}, + } + ), + ] + ) + + receipt = ProviderReceiptParser().parse( + arm="codex_cli", + stdout=stdout, + model_hint="o4-mini", + ) + + assert receipt is not None + assert receipt.cost_usd == 0.00154 + assert receipt.cost_source == "calculated_from_usage" + assert "thread-1" not in receipt.to_json() + + +def test_parse_claude_receipt_uses_provider_reported_cost() -> None: + stdout = "\n".join( + [ + json.dumps({"type": "system", "subtype": "init", "model": "claude-sonnet-4-6"}), + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "total_cost_usd": 0.012, + "usage": {"input_tokens": 100, "output_tokens": 20}, + } + ), + ] + ) + + receipt = ProviderReceiptParser().parse(arm="claude_code", stdout=stdout) + + assert receipt is not None + assert receipt.model == "claude-sonnet-4-6" + assert receipt.cost_usd == 0.012 + assert receipt.cost_source == "provider_reported" + assert "done" not in receipt.to_json() + + +def test_parse_morphic_receipt_requires_canonical_envelope() -> None: + stdout = json.dumps( + { + "type": "morphic_benchmark_receipt", + "success": True, + "model": "o4-mini", + "cost_usd": 0.02, + "usage": {"input_tokens": 120, "output_tokens": 30}, + } + ) + + receipt = ProviderReceiptParser().parse(arm="morphic_control", stdout=stdout) + + assert receipt is not None + assert receipt.cost_usd == 0.02 + assert receipt.provider == "morphic_control" + assert ProviderReceiptParser().parse(arm="morphic_control", stdout="not-json") is None + + +def test_receipt_rejects_wrong_source_negative_usage_and_tampered_codex_cost() -> None: + base = { + "provider": "codex_cli", + "success": True, + "model": "o4-mini", + "usage": {"input_tokens": 100, "output_tokens": 20}, + "cost_usd": 0.000198, + "cost_source": "calculated_from_usage", + } + with pytest.raises(ValueError, match="cost_source"): + ProviderReceipt.model_validate({**base, "cost_source": "provider_reported"}) + with pytest.raises(ValueError, match="non-negative"): + ProviderReceipt.model_validate({**base, "usage": {"input_tokens": -1}}) + with pytest.raises(ValueError, match="does not match usage"): + ProviderReceipt.model_validate({**base, "cost_usd": 1.0}) + + +def test_finalize_joins_machine_evidence_and_reviews() -> None: + results = finalize_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedEvidence.model_validate(_evidence()), + AdjudicationReviews.model_validate(_reviews()), + ) + payload = results.model_dump(mode="json") + + assert len(payload["observations"]) == 3 + assert payload["observations"][0] == { + "arm": "codex_cli", + "trial": 1, + "completed": True, + "accepted_patch": True, + "passed_checks": ["unit"], + "elapsed_seconds": 10.0, + "cost_usd": 0.000198, + "human_interventions": 0, + "recovery_attempted": False, + "recovery_succeeded": False, + "passed_handoff_assertions": ["decision"], + } + + +def test_finalized_json_is_deterministic() -> None: + args = ( + AgentCliManifest.model_validate(_manifest()), + RecordedEvidence.model_validate(_evidence()), + AdjudicationReviews.model_validate(_reviews()), + ) + + first = finalized_results_json(finalize_recorded_results(*args)) + second = finalized_results_json(finalize_recorded_results(*args)) + + assert first == second + assert "timestamp" not in json.loads(first) + assert first == json.dumps(json.loads(first), ensure_ascii=False, sort_keys=True) + + +@pytest.mark.parametrize( + ("mutate", "match"), + [ + (lambda evidence, reviews: evidence["trials"][0].update(receipt=None), "receipt"), + ( + lambda evidence, reviews: reviews["decisions"][0].update( + agent_argv_sha256="e" * 64 + ), + "fingerprint", + ), + (lambda evidence, reviews: reviews["decisions"].pop(), "missing review"), + ( + lambda evidence, reviews: evidence.update(authorized_cost_cap_usd=0.5), + "authorized cost cap", + ), + ( + lambda evidence, reviews: evidence["trials"][0]["receipt"].update( + parse_errors=1 + ), + "parse errors", + ), + ( + lambda evidence, reviews: evidence.update( + cost_collection="pending_adjudication" + ), + "not normalized_receipts", + ), + ], +) +def test_finalize_rejects_incomplete_or_inconsistent_campaign( + mutate: object, + match: str, +) -> None: + evidence = _evidence() + reviews = _reviews() + mutate(evidence, reviews) # type: ignore[operator] + + with pytest.raises(ValueError, match=match): + finalize_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedEvidence.model_validate(evidence), + AdjudicationReviews.model_validate(reviews), + ) + + +def test_finalize_rejects_acceptance_when_provider_failed() -> None: + evidence = _evidence() + evidence["trials"][0]["receipt"]["success"] = False # type: ignore[index] + + with pytest.raises(ValueError, match="accepted_patch requires completed"): + finalize_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedEvidence.model_validate(evidence), + AdjudicationReviews.model_validate(_reviews()), + ) + + +def test_finalize_rejects_successful_recovery_when_trial_still_failed() -> None: + evidence = _evidence() + reviews = _reviews() + evidence["trials"][0]["receipt"]["success"] = False # type: ignore[index] + reviews["decisions"][0].update( # type: ignore[index] + accepted_patch=False, + recovery_attempted=True, + recovery_succeeded=True, + ) + + with pytest.raises(ValueError, match="recovery_succeeded requires completed"): + finalize_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedEvidence.model_validate(evidence), + AdjudicationReviews.model_validate(reviews), + ) + + +def test_cli_finalizes_without_starting_an_agent(tmp_path: Path) -> None: + manifest = tmp_path / "manifest.json" + evidence = tmp_path / "evidence.json" + reviews = tmp_path / "reviews.json" + output = tmp_path / "results.json" + manifest.write_text(json.dumps(_manifest()), encoding="utf-8") + evidence.write_text(json.dumps(_evidence()), encoding="utf-8") + reviews.write_text(json.dumps(_reviews()), encoding="utf-8") + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-finalize", + "--manifest", + str(manifest), + "--evidence", + str(evidence), + "--reviews", + str(reviews), + "--output", + str(output), + "--json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.output)["observations"][0]["arm"] == "codex_cli" + assert len(json.loads(output.read_text(encoding="utf-8"))["observations"]) == 3 + + second = runner.invoke( + app, + [ + "benchmark", + "agent-cli-finalize", + "--manifest", + str(manifest), + "--evidence", + str(evidence), + "--reviews", + str(reviews), + "--output", + str(output), + ], + ) + assert second.exit_code == 1 + assert "already exists" in second.output diff --git a/tests/unit/benchmarks/test_agent_cli_checkpoint_registry.py b/tests/unit/benchmarks/test_agent_cli_checkpoint_registry.py new file mode 100644 index 0000000..5bd7983 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_checkpoint_registry.py @@ -0,0 +1,1898 @@ +"""Append-only checkpoint registry and authenticated peer exchange tests.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +import stat +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from typer.testing import CliRunner + +from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + build_benchmark_authority, +) +from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointAcknowledgementSigningRequest, + CheckpointExchangeSigningRequest, + CheckpointPeerCursorStore, + CheckpointPeerKeyDeclaration, + CheckpointPeerTrustDeclaration, + CheckpointRangeSigningRequest, + CheckpointRegistryStore, + SignedCheckpointExchangePacket, + SignedCheckpointRangeBundle, + build_checkpoint_acknowledgement_request, + build_checkpoint_exchange_request, + build_checkpoint_peer_trust, + build_checkpoint_range_request, + build_signed_checkpoint_acknowledgement, + build_signed_checkpoint_exchange_packet, + build_signed_checkpoint_range_bundle, + verify_checkpoint_acknowledgement, + verify_checkpoint_exchange_packet, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_gossip import ( + CheckpointGossipService, + fetch_checkpoint_gossip_status, + fetch_signed_checkpoint_range, + submit_signed_checkpoint_acknowledgement, +) +from benchmarks.agent_cli_gossip_sync import ( + CheckpointGossipSyncAuditStore, + CheckpointGossipSyncPolicy, + run_checkpoint_gossip_sync, +) +from benchmarks.agent_cli_gossip_transport import ( + CheckpointGossipServer, + send_checkpoint_gossip_request, +) +from benchmarks.agent_cli_peer_trust_ledger import ( + CheckpointPeerRotationSignature, + CheckpointPeerTrustGeneration, + build_checkpoint_peer_trust_ledger, + build_checkpoint_peer_trust_rotation_certificate, + build_checkpoint_peer_trust_rotation_template, +) +from benchmarks.agent_cli_transparency import ( + AuthorityRootGeneration, + SignedAuthorityRootLedger, + SignedTransparencyTreeHead, + TransparencyLogEntry, + build_authority_root_ledger_request, + build_transparency_consistency_proof, + build_transparency_log, + build_transparency_tree_head_request, +) +from benchmarks.agent_cli_witness import ( + TransparencyWitnessKeyDeclaration, + TransparencyWitnessSignature, + TransparencyWitnessTrustDeclaration, + build_transparency_witness_trust, + build_witness_checkpoint_bundle, + build_witness_checkpoint_template, +) +from interface.cli.main import app + +runner = CliRunner() + + +def _private(seed: int) -> Ed25519PrivateKey: + return Ed25519PrivateKey.from_private_bytes(bytes([seed]) * 32) + + +def _context(): + authority_private = _private(40) + authority = build_benchmark_authority( + BenchmarkAuthorityDeclaration( + schema_version=SCHEMA_VERSION, + authority_id="registry-test-root", + public_key_base64=base64.b64encode( + authority_private.public_key().public_bytes_raw() + ).decode(), + ) + ) + ledger_request = build_authority_root_ledger_request( + ( + AuthorityRootGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + authority=authority, + ), + ) + ) + ledger = SignedAuthorityRootLedger( + statement=ledger_request.statement, + signature_base64=base64.b64encode( + authority_private.sign(ledger_request.statement.signing_bytes()) + ).decode(), + ) + witness_keys = {f"witness-{index}": _private(40 + index) for index in range(1, 4)} + witness_trust = build_transparency_witness_trust( + TransparencyWitnessTrustDeclaration( + schema_version=SCHEMA_VERSION, + log_id="registry-log", + minimum_distinct_witnesses=2, + keys=tuple( + TransparencyWitnessKeyDeclaration( + witness_id=witness_id, + key_id=f"{witness_id}-key-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + ) + for witness_id, private_key in witness_keys.items() + ), + ) + ) + entries = tuple( + TransparencyLogEntry( + sequence=index, + kind="campaign_envelope", + artifact_sha256=f"{index + 1:064x}", + ) + for index in range(6) + ) + return authority_private, ledger, witness_keys, witness_trust, entries + + +def _signed_head(log, private_key, ledger): + request = build_transparency_tree_head_request(log, ledger) + return SignedTransparencyTreeHead( + statement=request.statement, + signature_base64=base64.b64encode( + private_key.sign(request.statement.signing_bytes()) + ).decode(), + ) + + +def _checkpoint(context, previous_size: int, current_size: int, *, entries=None): + authority_private, ledger, witness_keys, witness_trust, default_entries = context + selected_entries = entries or default_entries + current_log = build_transparency_log( + witness_trust.log_id, + selected_entries[:current_size], + ) + previous_log = build_transparency_log( + witness_trust.log_id, + selected_entries[:previous_size], + ) + proof = build_transparency_consistency_proof( + current_log, + previous_tree_head=_signed_head(previous_log, authority_private, ledger), + current_tree_head=_signed_head(current_log, authority_private, ledger), + authority_root_ledger=ledger, + ) + template = build_witness_checkpoint_template(witness_trust, proof, ledger) + signatures = tuple( + TransparencyWitnessSignature( + witness_id=request.witness_id, + key_id=f"{request.witness_id}-key-1", + signature_base64=base64.b64encode( + witness_keys[request.witness_id].sign( + request.statement.signing_bytes() + ) + ).decode(), + ) + for request in template.requests[:2] + ) + checkpoint = build_witness_checkpoint_bundle( + witness_trust, + proof, + ledger, + signatures, + ) + return proof, checkpoint + + +def _peer_trust(*, revoke_peer_2: bool = False): + peer_keys = {f"peer-{index}": _private(50 + index) for index in range(1, 3)} + declaration = CheckpointPeerTrustDeclaration( + schema_version=SCHEMA_VERSION, + registry_id="production-registry", + keys=tuple( + CheckpointPeerKeyDeclaration( + peer_id=peer_id, + key_id=f"{peer_id}-key-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + status=( + "revoked" if revoke_peer_2 and peer_id == "peer-2" else "active" + ), + ) + for peer_id, private_key in peer_keys.items() + ) + + ( + ( + CheckpointPeerKeyDeclaration( + peer_id="peer-2", + key_id="peer-2-key-2", + public_key_base64=base64.b64encode( + _private(59).public_key().public_bytes_raw() + ).decode(), + ), + ) + if revoke_peer_2 + else () + ), + ) + return peer_keys, build_checkpoint_peer_trust(declaration) + + +def test_registry_appends_replays_and_fsyncs_hash_chain(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + store = CheckpointRegistryStore( + tmp_path / "checkpoints.jsonl", + registry_id="production-registry", + ) + first_proof, first_checkpoint = _checkpoint(context, 1, 3) + second_proof, second_checkpoint = _checkpoint(context, 3, 5) + + first = store.append(first_proof, first_checkpoint, witness_trust, ledger) + second = store.append(second_proof, second_checkpoint, witness_trust, ledger) + snapshot = store.replay(witness_trust, ledger) + + assert snapshot.records == (first, second) + assert first.sequence == 0 + assert first.previous_record_sha256 is None + assert second.sequence == 1 + assert second.previous_record_sha256 == first.record_sha256 + assert snapshot.current_tree_size == 5 + assert snapshot.head_record_sha256 == second.record_sha256 + assert stat.S_IMODE(store.path.stat().st_mode) == 0o600 + + +def test_registry_duplicate_append_is_idempotent(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + store = CheckpointRegistryStore( + tmp_path / "checkpoints.jsonl", + registry_id="production-registry", + ) + proof, checkpoint = _checkpoint(context, 1, 3) + + first = store.append(proof, checkpoint, witness_trust, ledger) + second = store.append(proof, checkpoint, witness_trust, ledger) + + assert second == first + assert store.replay(witness_trust, ledger).record_count == 1 + + +def test_registry_serializes_concurrent_duplicate_appends(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + store = CheckpointRegistryStore( + tmp_path / "checkpoints.jsonl", + registry_id="production-registry", + ) + proof, checkpoint = _checkpoint(context, 1, 3) + + with ThreadPoolExecutor(max_workers=8) as executor: + records = tuple( + executor.map( + lambda _: store.append( + proof, + checkpoint, + witness_trust, + ledger, + ), + range(16), + ) + ) + + assert len({record.record_sha256 for record in records}) == 1 + assert store.replay(witness_trust, ledger).record_count == 1 + + +@pytest.mark.parametrize("mutation", ["sequence", "previous_hash", "record_hash"]) +def test_registry_replay_rejects_hash_chain_tampering( + tmp_path: Path, + mutation: str, +) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + store = CheckpointRegistryStore( + tmp_path / "checkpoints.jsonl", + registry_id="production-registry", + ) + first = _checkpoint(context, 1, 3) + second = _checkpoint(context, 3, 5) + store.append(*first, witness_trust, ledger) + store.append(*second, witness_trust, ledger) + lines = store.path.read_text(encoding="utf-8").splitlines() + payload = json.loads(lines[1]) + if mutation == "sequence": + payload["sequence"] = 4 + elif mutation == "previous_hash": + payload["previous_record_sha256"] = "f" * 64 + else: + payload["record_sha256"] = "f" * 64 + lines[1] = json.dumps(payload, sort_keys=True) + store.path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="sequence|hash chain|fingerprint"): + store.replay(witness_trust, ledger) + + +def test_registry_replay_rejects_truncated_tail(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + store = CheckpointRegistryStore( + tmp_path / "checkpoints.jsonl", + registry_id="production-registry", + ) + store.append(*_checkpoint(context, 1, 3), witness_trust, ledger) + store.path.write_text( + store.path.read_text(encoding="utf-8").rstrip("\n"), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="truncated final record"): + store.replay(witness_trust, ledger) + + +def test_registry_rejects_witnessed_split_view(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, entries = context + store = CheckpointRegistryStore( + tmp_path / "checkpoints.jsonl", + registry_id="production-registry", + ) + proof, checkpoint = _checkpoint(context, 1, 3) + store.append(proof, checkpoint, witness_trust, ledger) + alternate_entries = entries[:2] + ( + entries[2].model_copy(update={"artifact_sha256": "f" * 64}), + ) + entries[3:] + split_proof, split_checkpoint = _checkpoint( + context, + 1, + 3, + entries=alternate_entries, + ) + + with pytest.raises(ValueError, match="split-view checkpoint"): + store.append(split_proof, split_checkpoint, witness_trust, ledger) + + +def test_authenticated_peer_packet_imports_and_retries_idempotently( + tmp_path: Path, +) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + source = CheckpointRegistryStore( + tmp_path / "source.jsonl", + registry_id="production-registry", + ) + proof, checkpoint = _checkpoint(context, 1, 3) + record = source.append(proof, checkpoint, witness_trust, ledger) + peer_keys, peer_trust = _peer_trust() + request = build_checkpoint_exchange_request( + record, + peer_trust, + source_peer_id="peer-1", + ) + packet = build_signed_checkpoint_exchange_packet( + request, + record, + key_id="peer-1-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-1"].sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ) + target = CheckpointRegistryStore( + tmp_path / "target.jsonl", + registry_id="production-registry", + ) + + imported = target.import_packet( + packet, + peer_trust, + witness_trust, + ledger, + ) + retried = target.import_packet( + packet, + peer_trust, + witness_trust, + ledger, + ) + + assert imported == retried + assert target.replay(witness_trust, ledger).record_count == 1 + assert imported.checkpoint == record.checkpoint + + +def test_peer_packet_rejects_invalid_or_revoked_sender(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + source = CheckpointRegistryStore( + tmp_path / "source.jsonl", + registry_id="production-registry", + ) + proof, checkpoint = _checkpoint(context, 1, 3) + record = source.append(proof, checkpoint, witness_trust, ledger) + peer_keys, peer_trust = _peer_trust() + request = build_checkpoint_exchange_request( + record, + peer_trust, + source_peer_id="peer-2", + ) + packet = SignedCheckpointExchangePacket.model_validate( + build_signed_checkpoint_exchange_packet( + request, + record, + key_id="peer-2-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-2"].sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ).model_dump(mode="json") + ) + invalid_payload = packet.model_dump(mode="json", exclude={"packet_sha256"}) + invalid_payload["signature_base64"] = base64.b64encode(bytes(64)).decode() + invalid = SignedCheckpointExchangePacket( + **invalid_payload, + packet_sha256=hashlib.sha256( + json.dumps( + invalid_payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + ).hexdigest(), + ) + + with pytest.raises(ValueError, match="signature is invalid"): + verify_checkpoint_exchange_packet(invalid, peer_trust) + _, revoked_trust = _peer_trust(revoke_peer_2=True) + with pytest.raises(ValueError, match="active trusted key"): + verify_checkpoint_exchange_packet(packet, revoked_trust) + + +def test_registry_status_is_read_only_for_a_missing_path(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + path = tmp_path / "missing.jsonl" + + snapshot = CheckpointRegistryStore( + path, + registry_id="production-registry", + ).replay(witness_trust, ledger) + + assert snapshot.record_count == 0 + assert not path.exists() + + +def test_registry_rejects_stale_checkpoint(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + store = CheckpointRegistryStore( + tmp_path / "checkpoints.jsonl", + registry_id="production-registry", + ) + store.append(*_checkpoint(context, 1, 4), witness_trust, ledger) + + with pytest.raises(ValueError, match="stale checkpoint"): + store.append(*_checkpoint(context, 2, 3), witness_trust, ledger) + + +def test_peer_trust_requires_an_active_key_for_every_peer() -> None: + private_key = _private(61) + + with pytest.raises(ValueError, match="peer has no active key"): + build_checkpoint_peer_trust( + CheckpointPeerTrustDeclaration( + schema_version=SCHEMA_VERSION, + registry_id="production-registry", + keys=( + CheckpointPeerKeyDeclaration( + peer_id="retired-peer", + key_id="retired-peer-key-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + status="revoked", + ), + ), + ) + ) + + +def test_checkpoint_registry_cli_store_status_export_and_import( + tmp_path: Path, +) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + proof, checkpoint = _checkpoint(context, 1, 3) + proof_path = tmp_path / "proof.json" + checkpoint_path = tmp_path / "checkpoint.json" + witness_trust_path = tmp_path / "witness-trust.json" + ledger_path = tmp_path / "ledger.json" + registry_path = tmp_path / "source.jsonl" + for path, payload in ( + (proof_path, proof.to_json()), + (checkpoint_path, checkpoint.to_json()), + (witness_trust_path, witness_trust.to_json()), + (ledger_path, ledger.to_json()), + ): + path.write_text(payload, encoding="utf-8") + + stored = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-registry-store", + "--registry", + str(registry_path), + "--registry-id", + "production-registry", + "--consistency-proof", + str(proof_path), + "--witness-checkpoint", + str(checkpoint_path), + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(ledger_path), + "--json", + ], + ) + assert stored.exit_code == 0, stored.output + record = CheckpointRegistryStore( + registry_path, + registry_id="production-registry", + ).replay(witness_trust, ledger).records[0] + + status = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-registry-status", + "--registry", + str(registry_path), + "--registry-id", + "production-registry", + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(ledger_path), + "--json", + ], + ) + assert status.exit_code == 0, status.output + assert json.loads(status.output)["record_count"] == 1 + + peer_keys, peer_trust = _peer_trust() + peer_trust_path = tmp_path / "peer-trust.json" + peer_trust_path.write_text(peer_trust.to_json(), encoding="utf-8") + request_path = tmp_path / "exchange-request.json" + exported = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-registry-export-template", + "--registry", + str(registry_path), + "--registry-id", + "production-registry", + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(ledger_path), + "--peer-trust", + str(peer_trust_path), + "--source-peer-id", + "peer-1", + "--output", + str(request_path), + "--json", + ], + ) + assert exported.exit_code == 0, exported.output + request = CheckpointExchangeSigningRequest.model_validate_json( + request_path.read_text(encoding="utf-8") + ) + packet = build_signed_checkpoint_exchange_packet( + request, + record, + key_id="peer-1-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-1"].sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ) + packet_path = tmp_path / "packet.json" + packet_path.write_text(packet.to_json(), encoding="utf-8") + target_path = tmp_path / "target.jsonl" + + imported = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-registry-import", + "--registry", + str(target_path), + "--registry-id", + "production-registry", + "--packet", + str(packet_path), + "--peer-trust", + str(peer_trust_path), + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(ledger_path), + "--json", + ], + ) + assert imported.exit_code == 0, imported.output + assert json.loads(imported.output)["record_sha256"] == record.record_sha256 + + +def test_committed_peer_trust_template_and_cli(tmp_path: Path) -> None: + root = Path(__file__).parents[3] + declaration_path = ( + root + / "benchmarks/templates/agent_cli_checkpoint_peer_trust.example.json" + ) + declaration = CheckpointPeerTrustDeclaration.model_validate_json( + declaration_path.read_text(encoding="utf-8") + ) + trust = build_checkpoint_peer_trust(declaration) + output_path = tmp_path / "peer-trust.json" + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-peer-trust", + "--declaration", + str(declaration_path), + "--output", + str(output_path), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert output_path.read_text(encoding="utf-8") == trust.to_json() + assert {key.peer_id for key in trust.keys} == { + "registry-peer-a", + "registry-peer-b", + } + + +def _three_record_registry(tmp_path: Path, context): + _, ledger, _, witness_trust, _ = context + store = CheckpointRegistryStore( + tmp_path / "source-range.jsonl", + registry_id="production-registry", + ) + records = tuple( + store.append(*_checkpoint(context, previous, current), witness_trust, ledger) + for previous, current in ((1, 2), (2, 4), (4, 6)) + ) + return store, records + + +def _signed_range(records, peer_keys, peer_trust): + request = build_checkpoint_range_request( + records, + peer_trust, + source_peer_id="peer-1", + ) + return build_signed_checkpoint_range_bundle( + request, + records, + key_id="peer-1-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-1"].sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ) + + +def test_signed_range_imports_missing_suffix_and_retries_idempotently( + tmp_path: Path, +) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + _, records = _three_record_registry(tmp_path, context) + peer_keys, peer_trust = _peer_trust() + bundle = _signed_range(records[1:], peer_keys, peer_trust) + target = CheckpointRegistryStore( + tmp_path / "target-range.jsonl", + registry_id="production-registry", + ) + target.append( + records[0].consistency_proof, + records[0].checkpoint, + witness_trust, + ledger, + ) + + imported = target.import_range_bundle( + bundle, + peer_trust, + witness_trust, + ledger, + ) + retried = target.import_range_bundle( + bundle, + peer_trust, + witness_trust, + ledger, + ) + + assert imported.record_count == 3 + assert retried == imported + assert imported.records == records + + +def test_range_import_rejects_gap_without_partial_write(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + _, records = _three_record_registry(tmp_path, context) + peer_keys, peer_trust = _peer_trust() + gap_bundle = _signed_range(records[2:], peer_keys, peer_trust) + target = CheckpointRegistryStore( + tmp_path / "target-gap.jsonl", + registry_id="production-registry", + ) + target.append( + records[0].consistency_proof, + records[0].checkpoint, + witness_trust, + ledger, + ) + + with pytest.raises(ValueError, match="gap"): + target.import_range_bundle( + gap_bundle, + peer_trust, + witness_trust, + ledger, + ) + + assert target.replay(witness_trust, ledger).records == (records[0],) + + +def test_range_import_rejects_conflicting_overlap_without_partial_write( + tmp_path: Path, +) -> None: + context = _context() + _, ledger, _, witness_trust, entries = context + _, records = _three_record_registry(tmp_path, context) + target = CheckpointRegistryStore( + tmp_path / "target-fork.jsonl", + registry_id="production-registry", + ) + target.append( + records[0].consistency_proof, + records[0].checkpoint, + witness_trust, + ledger, + ) + alternate_entries = ( + entries[0], + entries[1].model_copy(update={"artifact_sha256": "f" * 64}), + ) + entries[2:] + alternate_proof, alternate_checkpoint = _checkpoint( + context, + 1, + 2, + entries=alternate_entries, + ) + alternate_store = CheckpointRegistryStore( + tmp_path / "alternate-range.jsonl", + registry_id="production-registry", + ) + alternate = alternate_store.append( + alternate_proof, + alternate_checkpoint, + witness_trust, + ledger, + ) + peer_keys, peer_trust = _peer_trust() + fork_bundle = _signed_range((alternate,), peer_keys, peer_trust) + + with pytest.raises(ValueError, match="conflicting range overlap"): + target.import_range_bundle( + fork_bundle, + peer_trust, + witness_trust, + ledger, + ) + + assert target.replay(witness_trust, ledger).records == (records[0],) + + +def test_range_bundle_rejects_invalid_signature(tmp_path: Path) -> None: + context = _context() + _, records = _three_record_registry(tmp_path, context) + _, peer_trust = _peer_trust() + request = build_checkpoint_range_request( + records, + peer_trust, + source_peer_id="peer-1", + ) + + with pytest.raises(ValueError, match="range signature is invalid"): + build_signed_checkpoint_range_bundle( + request, + records, + key_id="peer-1-key-1", + signature_base64=base64.b64encode(bytes(64)).decode(), + peer_trust=peer_trust, + ) + + +def test_signed_acknowledgement_advances_durable_peer_cursor( + tmp_path: Path, +) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + _, records = _three_record_registry(tmp_path, context) + peer_keys, peer_trust = _peer_trust() + first_bundle = _signed_range(records[:2], peer_keys, peer_trust) + target = CheckpointRegistryStore( + tmp_path / "ack-target.jsonl", + registry_id="production-registry", + ) + first_snapshot = target.import_range_bundle( + first_bundle, + peer_trust, + witness_trust, + ledger, + ) + invalid_snapshot = first_snapshot.model_copy(update={"record_count": 99}) + with pytest.raises(ValueError, match="record_count"): + build_checkpoint_acknowledgement_request( + first_bundle, + invalid_snapshot, + peer_trust, + acknowledging_peer_id="peer-2", + ) + request = build_checkpoint_acknowledgement_request( + first_bundle, + first_snapshot, + peer_trust, + acknowledging_peer_id="peer-2", + ) + with pytest.raises(ValueError, match="acknowledgement signature is invalid"): + build_signed_checkpoint_acknowledgement( + request, + key_id="peer-2-key-1", + signature_base64=base64.b64encode(bytes(64)).decode(), + peer_trust=peer_trust, + ) + acknowledgement = build_signed_checkpoint_acknowledgement( + request, + key_id="peer-2-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-2"].sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ) + cursors = CheckpointPeerCursorStore( + tmp_path / "peer-cursors.jsonl", + registry_id="production-registry", + ) + + first_cursor = cursors.append(acknowledgement, peer_trust) + retried = cursors.append(acknowledgement, peer_trust) + snapshot = cursors.replay(peer_trust) + + assert verify_checkpoint_acknowledgement( + acknowledgement, + peer_trust, + ) == acknowledgement.statement + assert retried == first_cursor + assert snapshot.cursor_count == 1 + assert snapshot.positions[0].source_peer_id == "peer-1" + assert snapshot.positions[0].acknowledging_peer_id == "peer-2" + assert snapshot.positions[0].acknowledged_record_sequence == 1 + assert stat.S_IMODE(cursors.path.stat().st_mode) == 0o600 + + payload = json.loads(cursors.path.read_text(encoding="utf-8")) + payload["cursor_record_sha256"] = "f" * 64 + cursors.path.write_text(json.dumps(payload, sort_keys=True) + "\n", encoding="utf-8") + with pytest.raises(ValueError, match="cursor record.*fingerprint"): + cursors.replay(peer_trust) + + +def test_peer_cursor_rejects_acknowledgement_regression(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, entries = context + _, records = _three_record_registry(tmp_path, context) + peer_keys, peer_trust = _peer_trust() + target = CheckpointRegistryStore( + tmp_path / "regression-target.jsonl", + registry_id="production-registry", + ) + full_bundle = _signed_range(records, peer_keys, peer_trust) + snapshot = target.import_range_bundle( + full_bundle, + peer_trust, + witness_trust, + ledger, + ) + + def acknowledgement_for(bundle, applied_snapshot): + request = build_checkpoint_acknowledgement_request( + bundle, + applied_snapshot, + peer_trust, + acknowledging_peer_id="peer-2", + ) + return build_signed_checkpoint_acknowledgement( + request, + key_id="peer-2-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-2"].sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ) + + cursor_store = CheckpointPeerCursorStore( + tmp_path / "regression-cursors.jsonl", + registry_id="production-registry", + ) + cursor_store.append(acknowledgement_for(full_bundle, snapshot), peer_trust) + older_bundle = _signed_range(records[:2], peer_keys, peer_trust) + older_target = CheckpointRegistryStore( + tmp_path / "older-regression-target.jsonl", + registry_id="production-registry", + ) + older_snapshot = older_target.import_range_bundle( + older_bundle, + peer_trust, + witness_trust, + ledger, + ) + + with pytest.raises(ValueError, match="cursor regression"): + cursor_store.append( + acknowledgement_for(older_bundle, older_snapshot), + peer_trust, + ) + + alternate_entries = ( + entries[0], + entries[1].model_copy(update={"artifact_sha256": "e" * 64}), + ) + entries[2:] + alternate_store = CheckpointRegistryStore( + tmp_path / "conflicting-cursor-source.jsonl", + registry_id="production-registry", + ) + alternate_records = tuple( + alternate_store.append( + *_checkpoint( + context, + previous, + current, + entries=alternate_entries, + ), + witness_trust, + ledger, + ) + for previous, current in ((1, 2), (2, 4), (4, 6)) + ) + alternate_bundle = _signed_range(alternate_records, peer_keys, peer_trust) + alternate_target = CheckpointRegistryStore( + tmp_path / "conflicting-cursor-target.jsonl", + registry_id="production-registry", + ) + alternate_snapshot = alternate_target.import_range_bundle( + alternate_bundle, + peer_trust, + witness_trust, + ledger, + ) + + with pytest.raises(ValueError, match="conflicting peer cursor"): + cursor_store.append( + acknowledgement_for(alternate_bundle, alternate_snapshot), + peer_trust, + ) + + +def test_checkpoint_range_and_cursor_cli_workflow(tmp_path: Path) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + source, records = _three_record_registry(tmp_path, context) + peer_keys, peer_trust = _peer_trust() + witness_trust_path = tmp_path / "range-witness-trust.json" + ledger_path = tmp_path / "range-ledger.json" + peer_trust_path = tmp_path / "range-peer-trust.json" + witness_trust_path.write_text(witness_trust.to_json(), encoding="utf-8") + ledger_path.write_text(ledger.to_json(), encoding="utf-8") + peer_trust_path.write_text(peer_trust.to_json(), encoding="utf-8") + range_request_path = tmp_path / "range-request.json" + + exported = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-range-export-template", + "--registry", + str(source.path), + "--registry-id", + "production-registry", + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(ledger_path), + "--peer-trust", + str(peer_trust_path), + "--source-peer-id", + "peer-1", + "--start-sequence", + "0", + "--max-records", + "3", + "--output", + str(range_request_path), + "--json", + ], + ) + assert exported.exit_code == 0, exported.output + range_request = CheckpointRangeSigningRequest.model_validate_json( + range_request_path.read_text(encoding="utf-8") + ) + bundle = build_signed_checkpoint_range_bundle( + range_request, + records, + key_id="peer-1-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-1"].sign(range_request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ) + bundle_path = tmp_path / "signed-range.json" + bundle_path.write_text(bundle.to_json(), encoding="utf-8") + target_path = tmp_path / "cli-range-target.jsonl" + + imported = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-range-import", + "--registry", + str(target_path), + "--registry-id", + "production-registry", + "--range-bundle", + str(bundle_path), + "--peer-trust", + str(peer_trust_path), + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(ledger_path), + "--json", + ], + ) + assert imported.exit_code == 0, imported.output + assert json.loads(imported.output)["record_count"] == 3 + + acknowledgement_request_path = tmp_path / "ack-request.json" + acknowledgement_template = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-acknowledgement-template", + "--registry", + str(target_path), + "--registry-id", + "production-registry", + "--range-bundle", + str(bundle_path), + "--peer-trust", + str(peer_trust_path), + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(ledger_path), + "--acknowledging-peer-id", + "peer-2", + "--output", + str(acknowledgement_request_path), + "--json", + ], + ) + assert acknowledgement_template.exit_code == 0, acknowledgement_template.output + acknowledgement_request = ( + CheckpointAcknowledgementSigningRequest.model_validate_json( + acknowledgement_request_path.read_text(encoding="utf-8") + ) + ) + acknowledgement = build_signed_checkpoint_acknowledgement( + acknowledgement_request, + key_id="peer-2-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-2"].sign( + acknowledgement_request.statement.signing_bytes() + ) + ).decode(), + peer_trust=peer_trust, + ) + acknowledgement_path = tmp_path / "signed-acknowledgement.json" + acknowledgement_path.write_text(acknowledgement.to_json(), encoding="utf-8") + cursor_path = tmp_path / "cli-peer-cursors.jsonl" + + stored = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-cursor-store", + "--cursor-ledger", + str(cursor_path), + "--registry-id", + "production-registry", + "--acknowledgement", + str(acknowledgement_path), + "--peer-trust", + str(peer_trust_path), + "--json", + ], + ) + assert stored.exit_code == 0, stored.output + cursor_status = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-cursor-status", + "--cursor-ledger", + str(cursor_path), + "--registry-id", + "production-registry", + "--peer-trust", + str(peer_trust_path), + "--json", + ], + ) + assert cursor_status.exit_code == 0, cursor_status.output + status_payload = json.loads(cursor_status.output) + assert status_payload["cursor_count"] == 1 + assert status_payload["positions"][0]["acknowledged_record_sequence"] == 2 + + +def test_peer_trust_ledger_replays_and_advances_historical_cursors( + tmp_path: Path, +) -> None: + context = _context() + _, authority_ledger, _, witness_trust, _ = context + _, records = _three_record_registry(tmp_path, context) + predecessor_keys, predecessor_trust = _peer_trust() + _, successor_trust = _peer_trust(revoke_peer_2=True) + template = build_checkpoint_peer_trust_rotation_template( + predecessor_trust, + successor_trust, + generation=2, + ) + requests = {request.peer_id: request for request in template.requests} + rotation = build_checkpoint_peer_trust_rotation_certificate( + template, + predecessor_trust, + successor_trust, + tuple( + CheckpointPeerRotationSignature( + peer_id=peer_id, + key_id=f"{peer_id}-key-1", + signature_base64=base64.b64encode( + predecessor_keys[peer_id].sign( + requests[peer_id].statement.signing_bytes() + ) + ).decode(), + ) + for peer_id in ("peer-1", "peer-2") + ), + ) + trust_ledger = build_checkpoint_peer_trust_ledger( + ( + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + trust=predecessor_trust, + ), + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=2, + trust=successor_trust, + rotation=rotation, + ), + ) + ) + target = CheckpointRegistryStore( + tmp_path / "rotation-target.jsonl", + registry_id="production-registry", + ) + + def signed_acknowledgement(bundle, snapshot, trust, key_id, private_key): + request = build_checkpoint_acknowledgement_request( + bundle, + snapshot, + trust, + acknowledging_peer_id="peer-2", + ) + return build_signed_checkpoint_acknowledgement( + request, + key_id=key_id, + signature_base64=base64.b64encode( + private_key.sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=trust, + ) + + first_bundle = _signed_range( + records[:2], + predecessor_keys, + predecessor_trust, + ) + first_snapshot = target.import_range_bundle( + first_bundle, + predecessor_trust, + witness_trust, + authority_ledger, + ) + first_acknowledgement = signed_acknowledgement( + first_bundle, + first_snapshot, + predecessor_trust, + "peer-2-key-1", + predecessor_keys["peer-2"], + ) + cursors = CheckpointPeerCursorStore( + tmp_path / "rotation-cursors.jsonl", + registry_id="production-registry", + ) + cursors.append(first_acknowledgement, predecessor_trust) + successor_keys = dict(predecessor_keys) + successor_keys["peer-2"] = _private(59) + second_bundle = _signed_range( + records[2:], + successor_keys, + successor_trust, + ) + second_snapshot = target.import_range_bundle( + second_bundle, + successor_trust, + witness_trust, + authority_ledger, + ) + second_acknowledgement = signed_acknowledgement( + second_bundle, + second_snapshot, + successor_trust, + "peer-2-key-2", + successor_keys["peer-2"], + ) + + with pytest.raises(ValueError, match="does not match peer trust"): + cursors.replay(successor_trust) + cursors.append(second_acknowledgement, trust_ledger) + cursor_snapshot = cursors.replay(trust_ledger) + + assert cursor_snapshot.cursor_count == 2 + assert cursor_snapshot.positions[0].acknowledged_record_sequence == 2 + trust_ledger_path = tmp_path / "rotation-trust-ledger.json" + trust_ledger_path.write_text(trust_ledger.to_json(), encoding="utf-8") + cursor_status = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-cursor-status", + "--cursor-ledger", + str(cursors.path), + "--registry-id", + "production-registry", + "--peer-trust-ledger", + str(trust_ledger_path), + "--json", + ], + ) + assert cursor_status.exit_code == 0, cursor_status.output + assert json.loads(cursor_status.output)["cursor_count"] == 2 + + +@pytest.mark.asyncio +async def test_authenticated_gossip_fetches_range_and_submits_acknowledgement( + tmp_path: Path, +) -> None: + context = _context() + _, ledger, _, witness_trust, _ = context + _, records = _three_record_registry(tmp_path, context) + peer_keys, peer_trust = _peer_trust() + bundle = _signed_range(records, peer_keys, peer_trust) + cursor_store = CheckpointPeerCursorStore( + tmp_path / "gossip-cursors.jsonl", + registry_id="production-registry", + ) + service = CheckpointGossipService( + registry_id="production-registry", + source_peer_id="peer-1", + range_bundles=(bundle,), + cursor_store=cursor_store, + peer_trust=peer_trust, + ) + descriptor_path = tmp_path / "gossip" / "peer-1.json" + + async with CheckpointGossipServer( + descriptor_path=descriptor_path, + registry_id="production-registry", + source_peer_id="peer-1", + handler=service, + max_requests=8, + ): + transport_operations: list[str] = [] + + async def recording_sender( + *, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: + transport_operations.append(operation) + return await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation=operation, + payload=payload, + ) + + status = await fetch_checkpoint_gossip_status( + descriptor_path=descriptor_path, + request_sender=recording_sender, + ) + assert status["source_peer_id"] == "peer-1" + fetched = await fetch_signed_checkpoint_range( + descriptor_path=descriptor_path, + start_sequence=0, + max_records=3, + peer_trust=peer_trust, + request_sender=recording_sender, + ) + assert fetched == bundle + + with pytest.raises(ValueError, match="does not match peer trust"): + _, wrong_trust = _peer_trust(revoke_peer_2=True) + await fetch_signed_checkpoint_range( + descriptor_path=descriptor_path, + start_sequence=0, + max_records=3, + peer_trust=wrong_trust, + ) + + target = CheckpointRegistryStore( + tmp_path / "gossip-target.jsonl", + registry_id="production-registry", + ) + snapshot = target.import_range_bundle( + fetched, + peer_trust, + witness_trust, + ledger, + ) + request = build_checkpoint_acknowledgement_request( + fetched, + snapshot, + peer_trust, + acknowledging_peer_id="peer-2", + ) + acknowledgement = build_signed_checkpoint_acknowledgement( + request, + key_id="peer-2-key-1", + signature_base64=base64.b64encode( + peer_keys["peer-2"].sign(request.statement.signing_bytes()) + ).decode(), + peer_trust=peer_trust, + ) + cursor_record = await submit_signed_checkpoint_acknowledgement( + descriptor_path=descriptor_path, + acknowledgement=acknowledgement, + request_sender=recording_sender, + ) + + assert cursor_record.acknowledgement == acknowledgement + assert cursor_store.replay(peer_trust).cursor_count == 1 + assert transport_operations == [ + "status", + "fetch_range", + "submit_acknowledgement", + ] + + fetched_path = tmp_path / "gossip-range.json" + cli_fetch = await asyncio.to_thread( + runner.invoke, + app, + [ + "benchmark", + "agent-cli-checkpoint-gossip-fetch", + "--descriptor", + str(descriptor_path), + "--peer-trust", + str(tmp_path / "peer-trust.json"), + "--start-sequence", + "0", + "--max-records", + "3", + "--output", + str(fetched_path), + "--json", + ], + ) + assert cli_fetch.exit_code == 2 + + peer_trust_path = tmp_path / "peer-trust.json" + peer_trust_path.write_text(peer_trust.to_json(), encoding="utf-8") + cli_fetch = await asyncio.to_thread( + runner.invoke, + app, + [ + "benchmark", + "agent-cli-checkpoint-gossip-fetch", + "--descriptor", + str(descriptor_path), + "--peer-trust", + str(peer_trust_path), + "--start-sequence", + "0", + "--max-records", + "3", + "--output", + str(fetched_path), + "--json", + ], + ) + assert cli_fetch.exit_code == 0, cli_fetch.output + assert SignedCheckpointRangeBundle.model_validate_json( + fetched_path.read_text(encoding="utf-8") + ) == bundle + cli_status = await asyncio.to_thread( + runner.invoke, + app, + [ + "benchmark", + "agent-cli-checkpoint-gossip-status", + "--descriptor", + str(descriptor_path), + "--json", + ], + ) + assert cli_status.exit_code == 0, cli_status.output + assert json.loads(cli_status.output)["available_ranges"][0][ + "range_bundle_sha256" + ] == bundle.range_bundle_sha256 + acknowledgement_path = tmp_path / "gossip-acknowledgement.json" + acknowledgement_path.write_text(acknowledgement.to_json(), encoding="utf-8") + cli_ack = await asyncio.to_thread( + runner.invoke, + app, + [ + "benchmark", + "agent-cli-checkpoint-gossip-ack", + "--descriptor", + str(descriptor_path), + "--acknowledgement", + str(acknowledgement_path), + "--json", + ], + ) + assert cli_ack.exit_code == 0, cli_ack.output + assert json.loads(cli_ack.output)["acknowledgement"] == ( + acknowledgement.model_dump(mode="json") + ) + assert cursor_store.replay(peer_trust).cursor_count == 1 + invalid_acknowledgement = acknowledgement.model_copy( + update={"signature_base64": base64.b64encode(bytes(64)).decode()} + ) + with pytest.raises(ValueError, match="fingerprint"): + await submit_signed_checkpoint_acknowledgement( + descriptor_path=descriptor_path, + acknowledgement=invalid_acknowledgement, + ) + with pytest.raises(RuntimeError, match="invalid_acknowledgement"): + await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation="submit_acknowledgement", + payload={ + "acknowledgement": invalid_acknowledgement.model_dump(mode="json") + }, + ) + assert cursor_store.replay(peer_trust).cursor_count == 1 + + bundles_path = tmp_path / "gossip-bundles.json" + bundles_path.write_text( + json.dumps({"bundles": [bundle.model_dump(mode="json")]}, sort_keys=True), + encoding="utf-8", + ) + serve_descriptor = tmp_path / "served-gossip.json" + serve_task = asyncio.create_task( + asyncio.to_thread( + runner.invoke, + app, + [ + "benchmark", + "agent-cli-checkpoint-gossip-serve", + "--descriptor", + str(serve_descriptor), + "--range-bundles", + str(bundles_path), + "--cursor-ledger", + str(tmp_path / "served-cursors.jsonl"), + "--registry-id", + "production-registry", + "--source-peer-id", + "peer-1", + "--peer-trust", + str(peer_trust_path), + "--max-requests", + "1", + "--lifetime-seconds", + "5", + ], + ) + ) + for _ in range(200): + if serve_descriptor.exists() or serve_task.done(): + break + await asyncio.sleep(0.01) + assert serve_descriptor.exists(), (await serve_task).output + served_status = await fetch_checkpoint_gossip_status( + descriptor_path=serve_descriptor + ) + serve_result = await asyncio.wait_for(serve_task, timeout=2) + + assert served_status["source_peer_id"] == "peer-1" + assert serve_result.exit_code == 0, serve_result.output + assert serve_descriptor.exists() is False + + +@pytest.mark.asyncio +async def test_gossip_sync_resumes_retries_pins_trust_and_rejects_tampering( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = _context() + _, authority_ledger, _, witness_trust, _ = context + _, records = _three_record_registry(tmp_path, context) + peer_keys, peer_trust = _peer_trust() + first_bundle = _signed_range(records[:2], peer_keys, peer_trust) + second_bundle = _signed_range(records[2:], peer_keys, peer_trust) + trust_ledger_v1 = build_checkpoint_peer_trust_ledger( + ( + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + trust=peer_trust, + ), + ) + ) + source_cursors = CheckpointPeerCursorStore( + tmp_path / "sync-source-cursors.jsonl", + registry_id="production-registry", + ) + service = CheckpointGossipService( + registry_id="production-registry", + source_peer_id="peer-1", + range_bundles=(first_bundle, second_bundle), + cursor_store=source_cursors, + peer_trust=trust_ledger_v1, + ) + descriptor_path = tmp_path / "sync-gossip.json" + target = CheckpointRegistryStore( + tmp_path / "sync-target.jsonl", + registry_id="production-registry", + ) + audit = CheckpointGossipSyncAuditStore( + tmp_path / "sync-audit.jsonl", + registry_id="production-registry", + source_peer_id="peer-1", + ) + policy = CheckpointGossipSyncPolicy( + max_rounds=4, + max_records=10, + max_attempts_per_request=2, + retry_delays_seconds=(0.0,), + ) + + async with CheckpointGossipServer( + descriptor_path=descriptor_path, + registry_id="production-registry", + source_peer_id="peer-1", + handler=service, + max_requests=32, + ): + first = await run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=target, + audit_store=audit, + peer_trust_ledger=trust_ledger_v1, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=policy, + ) + + assert first.stop_reason == "up_to_date" + assert first.ranges_imported == 2 + assert first.records_imported == 3 + assert first.retries == 0 + assert first.sync_policy_sha256 == policy.policy_sha256 + assert audit.replay(trust_ledger_v1).records[0].sync_policy_sha256 == ( + policy.policy_sha256 + ) + assert target.replay(witness_trust, authority_ledger).records == records + + injected_target = CheckpointRegistryStore( + tmp_path / "sync-injected-target.jsonl", + registry_id="production-registry", + ) + injected_audit = CheckpointGossipSyncAuditStore( + tmp_path / "sync-injected-audit.jsonl", + registry_id="production-registry", + source_peer_id="peer-1", + ) + injected_operations: list[str] = [] + + async def injected_sender( + *, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: + injected_operations.append(operation) + return await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation=operation, + payload=payload, + ) + + injected = await run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=injected_target, + audit_store=injected_audit, + peer_trust_ledger=trust_ledger_v1, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=policy, + request_sender=injected_sender, + ) + + assert injected.stop_reason == "up_to_date" + assert injected.records_imported == 3 + assert injected_operations == [ + "status", + "fetch_range", + "status", + "fetch_range", + "status", + ] + + from benchmarks import agent_cli_gossip_sync as sync_module + + original_status = sync_module.fetch_checkpoint_gossip_status + status_calls = 0 + + async def flaky_status(*, descriptor_path: Path): + nonlocal status_calls + status_calls += 1 + if status_calls == 1: + raise RuntimeError("synthetic transient transport failure") + return await original_status(descriptor_path=descriptor_path) + + monkeypatch.setattr(sync_module, "fetch_checkpoint_gossip_status", flaky_status) + resumed = await run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=target, + audit_store=audit, + peer_trust_ledger=trust_ledger_v1, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=policy, + ) + monkeypatch.setattr( + sync_module, + "fetch_checkpoint_gossip_status", + original_status, + ) + + assert resumed.stop_reason == "up_to_date" + assert resumed.ranges_imported == 0 + assert resumed.retries == 1 + assert target.replay(witness_trust, authority_ledger).record_count == 3 + + _, successor_trust = _peer_trust(revoke_peer_2=True) + template = build_checkpoint_peer_trust_rotation_template( + peer_trust, + successor_trust, + generation=2, + ) + requests = {request.peer_id: request for request in template.requests} + rotation = build_checkpoint_peer_trust_rotation_certificate( + template, + peer_trust, + successor_trust, + tuple( + CheckpointPeerRotationSignature( + peer_id=peer_id, + key_id=f"{peer_id}-key-1", + signature_base64=base64.b64encode( + peer_keys[peer_id].sign( + requests[peer_id].statement.signing_bytes() + ) + ).decode(), + ) + for peer_id in ("peer-1", "peer-2") + ), + ) + trust_ledger_v2 = build_checkpoint_peer_trust_ledger( + ( + trust_ledger_v1.generations[0], + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=2, + trust=successor_trust, + rotation=rotation, + ), + ) + ) + advanced = await run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=target, + audit_store=audit, + peer_trust_ledger=trust_ledger_v2, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=policy, + ) + + assert advanced.stop_reason == "up_to_date" + assert advanced.peer_trust_generation == 2 + + recovery_target = CheckpointRegistryStore( + tmp_path / "sync-recovery-target.jsonl", + registry_id="production-registry", + ) + recovery_target.import_range_bundle( + first_bundle, + peer_trust, + witness_trust, + authority_ledger, + ) + recovery_audit = CheckpointGossipSyncAuditStore( + tmp_path / "sync-recovery-audit.jsonl", + registry_id="production-registry", + source_peer_id="peer-1", + ) + recovered = await run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=recovery_target, + audit_store=recovery_audit, + peer_trust_ledger=trust_ledger_v2, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=policy, + ) + + assert recovered.stop_reason == "up_to_date" + assert recovered.records_imported == 1 + assert recovery_audit.replay(trust_ledger_v2).records[0].event == "recovered" + + budget_target = CheckpointRegistryStore( + tmp_path / "sync-budget-target.jsonl", + registry_id="production-registry", + ) + budget_audit = CheckpointGossipSyncAuditStore( + tmp_path / "sync-budget-audit.jsonl", + registry_id="production-registry", + source_peer_id="peer-1", + ) + budgeted = await run_checkpoint_gossip_sync( + descriptor_path=descriptor_path, + registry_store=budget_target, + audit_store=budget_audit, + peer_trust_ledger=trust_ledger_v2, + witness_trust=witness_trust, + authority_root_ledger=authority_ledger, + policy=CheckpointGossipSyncPolicy( + max_rounds=4, + max_records=1, + max_attempts_per_request=1, + retry_delays_seconds=(), + ), + ) + + assert budgeted.stop_reason == "record_budget_exhausted" + assert budget_target.replay(witness_trust, authority_ledger).record_count == 0 + + audit_snapshot = audit.replay(trust_ledger_v2) + assert audit_snapshot.peer_trust_generation == 2 + assert {record.event for record in audit_snapshot.records} >= { + "imported", + "retry", + "stopped", + "trust_advanced", + } + with pytest.raises(ValueError, match="rollback"): + audit.replay(trust_ledger_v1) + + trust_ledger_path = tmp_path / "sync-trust-ledger.json" + witness_trust_path = tmp_path / "sync-witness-trust.json" + authority_ledger_path = tmp_path / "sync-authority-ledger.json" + trust_ledger_path.write_text(trust_ledger_v2.to_json(), encoding="utf-8") + witness_trust_path.write_text(witness_trust.to_json(), encoding="utf-8") + authority_ledger_path.write_text(authority_ledger.to_json(), encoding="utf-8") + cli_target = tmp_path / "sync-cli-target.jsonl" + cli_audit = tmp_path / "sync-cli-audit.jsonl" + cli_descriptor = tmp_path / "sync-cli-gossip.json" + cli_service = CheckpointGossipService( + registry_id="production-registry", + source_peer_id="peer-1", + range_bundles=(first_bundle, second_bundle), + cursor_store=CheckpointPeerCursorStore( + tmp_path / "sync-cli-source-cursors.jsonl", + registry_id="production-registry", + ), + peer_trust=trust_ledger_v2, + ) + async with CheckpointGossipServer( + descriptor_path=cli_descriptor, + registry_id="production-registry", + source_peer_id="peer-1", + handler=cli_service, + max_requests=8, + ): + cli_result = await asyncio.to_thread( + runner.invoke, + app, + [ + "benchmark", + "agent-cli-checkpoint-gossip-sync", + "--descriptor", + str(cli_descriptor), + "--registry", + str(cli_target), + "--sync-audit", + str(cli_audit), + "--registry-id", + "production-registry", + "--source-peer-id", + "peer-1", + "--peer-trust-ledger", + str(trust_ledger_path), + "--witness-trust", + str(witness_trust_path), + "--authority-root-ledger", + str(authority_ledger_path), + "--max-rounds", + "4", + "--max-records", + "10", + "--max-attempts", + "2", + "--json", + ], + ) + + assert cli_result.exit_code == 0, cli_result.output + assert json.loads(cli_result.output)["stop_reason"] == "up_to_date" + assert CheckpointRegistryStore( + cli_target, + registry_id="production-registry", + ).replay(witness_trust, authority_ledger).record_count == 3 + + lines = audit.path.read_text(encoding="utf-8").splitlines() + tampered = json.loads(lines[-1]) + tampered["record_sha256"] = "f" * 64 + lines[-1] = json.dumps(tampered, sort_keys=True) + audit.path.write_text("\n".join(lines) + "\n", encoding="utf-8") + with pytest.raises(ValueError, match="fingerprint"): + audit.replay(trust_ledger_v2) diff --git a/tests/unit/benchmarks/test_agent_cli_comparison.py b/tests/unit/benchmarks/test_agent_cli_comparison.py new file mode 100644 index 0000000..a1f48f2 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_comparison.py @@ -0,0 +1,178 @@ +"""Tests for the recorded same-task agent CLI comparison benchmark.""" + +from __future__ import annotations + +import json + +import pytest + +from benchmarks.agent_cli_comparison import ( + AgentCliManifest, + RecordedResults, + evaluate_recorded_results, +) + + +def _manifest() -> dict[str, object]: + return { + "schema_version": 1, + "benchmark_id": "same-task-001", + "task": { + "id": "task-001", + "goal": "Implement the same tested change", + "workspace_revision": "abc123", + "checks": ["unit", "ruff"], + "handoff_assertions": ["decision", "failure"], + }, + "arms": ["codex_cli", "claude_code", "morphic_control"], + "repetitions": 2, + } + + +def _observations() -> dict[str, object]: + rows: list[dict[str, object]] = [] + values = { + "codex_cli": [ + (True, True, ["unit", "ruff"], 120.0, 0.20, 1), + (False, False, [], 180.0, 0.10, 2), + ], + "claude_code": [ + (True, True, ["unit"], 90.0, 0.30, 1), + (True, False, ["unit", "ruff"], 110.0, 0.40, 1), + ], + "morphic_control": [ + (True, True, ["unit", "ruff"], 105.0, 0.25, 0), + (True, True, ["unit", "ruff"], 105.0, 0.25, 0), + ], + } + for arm, trials in values.items(): + for trial, (completed, accepted, checks, elapsed, cost, interventions) in enumerate( + trials, start=1 + ): + rows.append( + { + "arm": arm, + "trial": trial, + "completed": completed, + "accepted_patch": accepted, + "passed_checks": checks, + "elapsed_seconds": elapsed, + "cost_usd": cost, + "human_interventions": interventions, + "recovery_attempted": trial == 2, + "recovery_succeeded": arm != "codex_cli" and trial == 2, + "passed_handoff_assertions": ( + ["decision", "failure"] if arm == "morphic_control" else ["decision"] + ), + } + ) + return { + "schema_version": 1, + "benchmark_id": "same-task-001", + "task_id": "task-001", + "observations": rows, + } + + +def test_evaluate_same_task_results_without_a_composite_score() -> None: + report = evaluate_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedResults.model_validate(_observations()), + ).to_dict() + + assert report["benchmark_id"] == "same-task-001" + assert report["observation_count"] == 6 + assert "overall_score" not in report + assert report["arms"]["codex_cli"] == { + "accepted_patch_rate": 0.5, + "completion_rate": 0.5, + "context_handoff_score": 0.5, + "mean_cost_usd": 0.15, + "mean_human_interventions": 1.5, + "median_elapsed_seconds": 150.0, + "recovery_rate": 0.0, + "verification_rate": 0.5, + } + assert report["leaders"]["accepted_patch_rate"] == ["morphic_control"] + assert report["leaders"]["median_elapsed_seconds"] == ["claude_code"] + assert report["leaders"]["mean_human_interventions"] == ["morphic_control"] + assert report["leaders"]["context_handoff_score"] == ["morphic_control"] + + +def test_report_json_is_deterministic_and_has_no_timestamp() -> None: + manifest = AgentCliManifest.model_validate(_manifest()) + results = RecordedResults.model_validate(_observations()) + + first = evaluate_recorded_results(manifest, results).to_json() + second = evaluate_recorded_results(manifest, results).to_json() + + assert first == second + assert "timestamp" not in json.loads(first) + assert first == json.dumps(json.loads(first), ensure_ascii=False, sort_keys=True) + + +@pytest.mark.parametrize( + ("mutate", "match"), + [ + (lambda data: data["observations"].pop(), "missing observations"), + ( + lambda data: data["observations"].append(dict(data["observations"][0])), + "duplicate observation", + ), + (lambda data: data.update(task_id="other-task"), "task_id"), + ], +) +def test_rejects_incomplete_duplicate_or_mismatched_results(mutate: object, match: str) -> None: + raw = _observations() + mutate(raw) # type: ignore[operator] + + with pytest.raises(ValueError, match=match): + evaluate_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedResults.model_validate(raw), + ) + + +def test_manifest_requires_the_three_comparison_arms() -> None: + raw = _manifest() + raw["arms"] = ["codex_cli", "morphic_control"] + + with pytest.raises(ValueError, match="exactly"): + AgentCliManifest.model_validate(raw) + + +def test_rejects_invalid_recovery_claim() -> None: + raw = _observations() + raw["observations"][0]["recovery_succeeded"] = True # type: ignore[index] + + with pytest.raises(ValueError, match="recovery_succeeded"): + RecordedResults.model_validate(raw) + + +def test_rejects_non_finite_measurements() -> None: + raw = _observations() + raw["observations"][0]["elapsed_seconds"] = float("inf") # type: ignore[index] + + with pytest.raises(ValueError, match="finite number"): + RecordedResults.model_validate(raw) + + +def test_rejects_undeclared_checks_and_handoff_assertions() -> None: + raw = _observations() + raw["observations"][0]["passed_checks"] = ["not-declared"] # type: ignore[index] + + with pytest.raises(ValueError, match="undeclared checks"): + evaluate_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedResults.model_validate(raw), + ) + + raw = _observations() + raw["observations"][0]["passed_handoff_assertions"] = [ # type: ignore[index] + "not-declared" + ] + with pytest.raises(ValueError, match="undeclared handoff assertions"): + evaluate_recorded_results( + AgentCliManifest.model_validate(_manifest()), + RecordedResults.model_validate(raw), + ) diff --git a/tests/unit/benchmarks/test_agent_cli_gossip_tls.py b/tests/unit/benchmarks/test_agent_cli_gossip_tls.py new file mode 100644 index 0000000..b856919 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_gossip_tls.py @@ -0,0 +1,634 @@ +"""Peer-signed TLS identity and mutual-auth gossip transport tests.""" + +from __future__ import annotations + +import asyncio +import base64 +import ipaddress +import json +import stat +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from click import unstyle +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID +from typer.testing import CliRunner + +from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerKeyDeclaration, + CheckpointPeerTrustDeclaration, + build_checkpoint_peer_trust, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_gossip_tls_identity import ( + CheckpointPeerTlsEnrollment, + CheckpointPeerTlsRevocation, + CheckpointPeerTlsRevocationTemplate, + CheckpointPeerTlsTrust, + build_checkpoint_peer_tls_enrollment_template, + build_checkpoint_peer_tls_trust, + build_signed_checkpoint_peer_tls_enrollment, + build_signed_checkpoint_peer_tls_revocation, +) +from benchmarks.agent_cli_gossip_tls_transport import ( + CheckpointMutualTlsGossipClient, + CheckpointMutualTlsGossipServer, + send_checkpoint_mtls_gossip_request, +) +from interface.cli.main import app + +runner = CliRunner() + + +def _private(seed: int) -> Ed25519PrivateKey: + return Ed25519PrivateKey.from_private_bytes(bytes([seed]) * 32) + + +def _write_private_key(path: Path, private_key: Ed25519PrivateKey) -> None: + path.write_bytes( + private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + path.chmod(0o600) + + +def _certificate_authority() -> tuple[Ed25519PrivateKey, x509.Certificate]: + private_key = _private(80) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Morphic Test CA")]) + certificate = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(private_key.public_key()) + .serial_number(1) + .not_valid_before(datetime(2025, 1, 1, tzinfo=UTC)) + .not_valid_after(datetime(2035, 1, 1, tzinfo=UTC)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=True, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(private_key, algorithm=None) + ) + return private_key, certificate + + +def _leaf_certificate( + *, + common_name: str, + serial_number: int, + ca_private_key: Ed25519PrivateKey, + ca_certificate: x509.Certificate, +) -> tuple[Ed25519PrivateKey, x509.Certificate]: + private_key = _private(80 + serial_number) + certificate = ( + x509.CertificateBuilder() + .subject_name( + x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, common_name)]) + ) + .issuer_name(ca_certificate.subject) + .public_key(private_key.public_key()) + .serial_number(serial_number) + .not_valid_before(datetime(2025, 1, 1, tzinfo=UTC)) + .not_valid_after(datetime(2030, 1, 1, tzinfo=UTC)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.SubjectAlternativeName( + [ + x509.DNSName("localhost"), + x509.IPAddress(ipaddress.ip_address("127.0.0.1")), + ] + ), + critical=False, + ) + .add_extension( + x509.ExtendedKeyUsage( + [ExtendedKeyUsageOID.CLIENT_AUTH, ExtendedKeyUsageOID.SERVER_AUTH] + ), + critical=False, + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(ca_private_key, algorithm=None) + ) + return private_key, certificate + + +def _write_certificate(path: Path, certificate: x509.Certificate) -> None: + path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + + +def _identity_context(): + identity_keys = { + "server-peer": _private(91), + "client-peer": _private(92), + } + trust = build_checkpoint_peer_trust( + CheckpointPeerTrustDeclaration( + schema_version=SCHEMA_VERSION, + registry_id="tls-registry", + keys=tuple( + CheckpointPeerKeyDeclaration( + peer_id=peer_id, + key_id=f"{peer_id}-identity-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + ) + for peer_id, private_key in identity_keys.items() + ), + ) + ) + return identity_keys, trust + + +def _enrollment( + certificate: x509.Certificate, + *, + peer_id: str, + identity_key: Ed25519PrivateKey, + peer_trust, + generation: int = 1, + predecessor: CheckpointPeerTlsEnrollment | None = None, +) -> CheckpointPeerTlsEnrollment: + certificate_pem = certificate.public_bytes(serialization.Encoding.PEM) + template = build_checkpoint_peer_tls_enrollment_template( + certificate_pem, + peer_trust, + peer_id=peer_id, + generation=generation, + predecessor=predecessor, + ) + return build_signed_checkpoint_peer_tls_enrollment( + template, + certificate_pem, + peer_trust, + key_id=f"{peer_id}-identity-1", + signature_base64=base64.b64encode( + identity_key.sign(template.statement.signing_bytes()) + ).decode(), + predecessor=predecessor, + ) + + +class _RecordingHandler: + def __init__(self) -> None: + self.requests: list[tuple[str, dict[str, object]]] = [] + + async def dispatch( + self, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: + self.requests.append((operation, payload)) + return {"operation": operation, "payload": payload} + + +@pytest.mark.asyncio +async def test_peer_signed_tls_rotation_and_mutual_tls_transport( + tmp_path: Path, +) -> None: + identity_keys, peer_trust = _identity_context() + ca_key, ca_certificate = _certificate_authority() + server_key_v1, server_certificate_v1 = _leaf_certificate( + common_name="server-peer-v1", + serial_number=1, + ca_private_key=ca_key, + ca_certificate=ca_certificate, + ) + server_key_v2, server_certificate_v2 = _leaf_certificate( + common_name="server-peer-v2", + serial_number=2, + ca_private_key=ca_key, + ca_certificate=ca_certificate, + ) + client_key, client_certificate = _leaf_certificate( + common_name="client-peer", + serial_number=3, + ca_private_key=ca_key, + ca_certificate=ca_certificate, + ) + server_enrollment_v1 = _enrollment( + server_certificate_v1, + peer_id="server-peer", + identity_key=identity_keys["server-peer"], + peer_trust=peer_trust, + ) + server_enrollment_v2 = _enrollment( + server_certificate_v2, + peer_id="server-peer", + identity_key=identity_keys["server-peer"], + peer_trust=peer_trust, + generation=2, + predecessor=server_enrollment_v1, + ) + client_enrollment = _enrollment( + client_certificate, + peer_id="client-peer", + identity_key=identity_keys["client-peer"], + peer_trust=peer_trust, + ) + server_template = build_checkpoint_peer_tls_enrollment_template( + server_certificate_v1.public_bytes(serialization.Encoding.PEM), + peer_trust, + peer_id="server-peer", + generation=1, + ) + with pytest.raises(ValueError, match="signature is invalid"): + build_signed_checkpoint_peer_tls_enrollment( + server_template, + server_certificate_v1.public_bytes(serialization.Encoding.PEM), + peer_trust, + key_id="server-peer-identity-1", + signature_base64=base64.b64encode( + identity_keys["client-peer"].sign( + server_template.statement.signing_bytes() + ) + ).decode(), + ) + tls_trust = build_checkpoint_peer_tls_trust( + peer_trust, + (server_enrollment_v1, server_enrollment_v2, client_enrollment), + ) + + assert tls_trust.active_enrollment("server-peer") == server_enrollment_v2 + assert tls_trust.active_enrollment("client-peer") == client_enrollment + with pytest.raises(ValueError, match="predecessor"): + build_checkpoint_peer_tls_trust( + peer_trust, + (server_enrollment_v2, client_enrollment), + ) + + ca_path = tmp_path / "ca.pem" + server_certificate_path = tmp_path / "server-v2.pem" + server_key_path = tmp_path / "server-v2-key.pem" + old_server_certificate_path = tmp_path / "server-v1.pem" + old_server_key_path = tmp_path / "server-v1-key.pem" + client_certificate_path = tmp_path / "client.pem" + client_key_path = tmp_path / "client-key.pem" + _write_certificate(ca_path, ca_certificate) + _write_certificate(server_certificate_path, server_certificate_v2) + _write_private_key(server_key_path, server_key_v2) + _write_certificate(old_server_certificate_path, server_certificate_v1) + _write_private_key(old_server_key_path, server_key_v1) + _write_certificate(client_certificate_path, client_certificate) + _write_private_key(client_key_path, client_key) + + with pytest.raises(ValueError, match="active.*certificate pin"): + CheckpointMutualTlsGossipServer( + descriptor_path=tmp_path / "old-server.json", + bind_host="127.0.0.1", + advertised_host="127.0.0.1", + registry_id="tls-registry", + server_peer_id="server-peer", + handler=_RecordingHandler(), + tls_trust=tls_trust, + certificate_path=old_server_certificate_path, + private_key_path=old_server_key_path, + certificate_authority_path=ca_path, + allowed_client_addresses=frozenset({"127.0.0.1"}), + ) + + handler = _RecordingHandler() + descriptor_path = tmp_path / "remote-gossip.json" + async with CheckpointMutualTlsGossipServer( + descriptor_path=descriptor_path, + bind_host="127.0.0.1", + advertised_host="127.0.0.1", + registry_id="tls-registry", + server_peer_id="server-peer", + handler=handler, + tls_trust=tls_trust, + certificate_path=server_certificate_path, + private_key_path=server_key_path, + certificate_authority_path=ca_path, + allowed_client_addresses=frozenset({"127.0.0.1"}), + max_requests=4, + ) as server: + descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) + assert descriptor["transport"] == "mtls" + assert descriptor["protocol_version"] == 2 + assert descriptor["server_peer_id"] == "server-peer" + assert "token" not in descriptor + assert stat.S_IMODE(descriptor_path.stat().st_mode) == 0o600 + + response = await send_checkpoint_mtls_gossip_request( + descriptor_path=descriptor_path, + client_peer_id="client-peer", + tls_trust=tls_trust, + certificate_path=client_certificate_path, + private_key_path=client_key_path, + certificate_authority_path=ca_path, + server_hostname="localhost", + allowed_server_addresses=frozenset({"127.0.0.1"}), + operation="status", + payload={"cursor": 3}, + client_nonce=bytes(range(32)), + ) + + assert response == {"operation": "status", "payload": {"cursor": 3}} + assert handler.requests == [("status", {"cursor": 3})] + assert server.completed_requests == 1 + with pytest.raises(RuntimeError, match="replayed_nonce"): + await send_checkpoint_mtls_gossip_request( + descriptor_path=descriptor_path, + client_peer_id="client-peer", + tls_trust=tls_trust, + certificate_path=client_certificate_path, + private_key_path=client_key_path, + certificate_authority_path=ca_path, + server_hostname="localhost", + allowed_server_addresses=frozenset({"127.0.0.1"}), + operation="status", + payload={}, + client_nonce=bytes(range(32)), + ) + with pytest.raises(RuntimeError, match="TLS|certificate|endpoint"): + await send_checkpoint_mtls_gossip_request( + descriptor_path=descriptor_path, + client_peer_id="client-peer", + tls_trust=tls_trust, + certificate_path=client_certificate_path, + private_key_path=client_key_path, + certificate_authority_path=ca_path, + server_hostname="wrong.example", + allowed_server_addresses=frozenset({"127.0.0.1"}), + operation="status", + payload={}, + ) + with pytest.raises(ValueError, match="allowlist"): + await send_checkpoint_mtls_gossip_request( + descriptor_path=descriptor_path, + client_peer_id="client-peer", + tls_trust=tls_trust, + certificate_path=client_certificate_path, + private_key_path=client_key_path, + certificate_authority_path=ca_path, + server_hostname="localhost", + allowed_server_addresses=frozenset({"192.0.2.1"}), + operation="status", + payload={}, + ) + + reusable_client = CheckpointMutualTlsGossipClient( + descriptor_path=descriptor_path, + client_peer_id="client-peer", + tls_trust=tls_trust, + certificate_path=client_certificate_path, + private_key_path=client_key_path, + certificate_authority_path=ca_path, + server_hostname="localhost", + allowed_server_addresses=frozenset({"127.0.0.1"}), + ) + reusable_response = await reusable_client( + operation="fetch_range", + payload={"start_sequence": 3}, + ) + assert reusable_response == { + "operation": "fetch_range", + "payload": {"start_sequence": 3}, + } + + tls_trust_path = tmp_path / "tls-trust.json" + tls_trust_path.write_text(tls_trust.to_json(), encoding="utf-8") + (tmp_path / "peer-trust.json").write_text( + peer_trust.to_json(), encoding="utf-8" + ) + cli_status = await asyncio.to_thread( + runner.invoke, + app, + [ + "benchmark", + "agent-cli-checkpoint-gossip-mtls-status", + "--descriptor", + str(descriptor_path), + "--tls-trust", + str(tls_trust_path), + "--peer-trust", + str(tmp_path / "peer-trust.json"), + "--client-peer-id", + "client-peer", + "--certificate", + str(client_certificate_path), + "--private-key", + str(client_key_path), + "--certificate-authority", + str(ca_path), + "--server-hostname", + "localhost", + "--allow-server-address", + "127.0.0.1", + "--json", + ], + ) + assert cli_status.exit_code == 0, cli_status.output + assert json.loads(cli_status.output) == { + "operation": "status", "payload": {}, "tls_expiry_warnings": [] + } + + reader, writer = await asyncio.open_connection( + descriptor["host"], + descriptor["port"], + ) + writer.write(b'{"operation":"status"}\n') + await writer.drain() + assert await asyncio.wait_for(reader.read(), timeout=1.0) == b"" + writer.close() + await writer.wait_closed() + + assert descriptor_path.exists() is False + assert server.active_client_count == 0 + + +def test_tls_identity_cli_is_private_key_free(tmp_path: Path) -> None: + identity_keys, peer_trust = _identity_context() + ca_key, ca_certificate = _certificate_authority() + _, server_certificate = _leaf_certificate( + common_name="server-peer", + serial_number=4, + ca_private_key=ca_key, + ca_certificate=ca_certificate, + ) + peer_trust_path = tmp_path / "peer-trust.json" + certificate_path = tmp_path / "server.pem" + template_path = tmp_path / "tls-enrollment-template.json" + peer_trust_path.write_text(peer_trust.to_json(), encoding="utf-8") + _write_certificate(certificate_path, server_certificate) + + template_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-peer-tls-enrollment-template", + "--certificate", + str(certificate_path), + "--peer-trust", + str(peer_trust_path), + "--peer-id", + "server-peer", + "--generation", + "1", + "--output", + str(template_path), + "--json", + ], + ) + + assert template_result.exit_code == 0, template_result.output + assert "private" not in template_path.read_text(encoding="utf-8").lower() + template = build_checkpoint_peer_tls_enrollment_template( + certificate_path.read_bytes(), + peer_trust, + peer_id="server-peer", + generation=1, + ) + enrollment = build_signed_checkpoint_peer_tls_enrollment( + template, + certificate_path.read_bytes(), + peer_trust, + key_id="server-peer-identity-1", + signature_base64=base64.b64encode( + identity_keys["server-peer"].sign(template.statement.signing_bytes()) + ).decode(), + ) + enrollment_path = tmp_path / "tls-enrollment.json" + enrollment_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-peer-tls-enrollment", + "--template", + str(template_path), + "--certificate", + str(certificate_path), + "--peer-trust", + str(peer_trust_path), + "--key-id", + "server-peer-identity-1", + "--signature-base64", + enrollment.signature_base64, + "--output", + str(enrollment_path), + "--json", + ], + ) + assert enrollment_result.exit_code == 0, enrollment_result.output + assert CheckpointPeerTlsEnrollment.model_validate_json( + enrollment_path.read_text(encoding="utf-8") + ) == enrollment + enrollments_path = tmp_path / "tls-enrollments.json" + enrollments_path.write_text( + json.dumps( + {"enrollments": [enrollment.model_dump(mode="json")]}, + sort_keys=True, + ), + encoding="utf-8", + ) + tls_trust_path = tmp_path / "tls-trust.json" + trust_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-peer-tls-trust", + "--peer-trust", + str(peer_trust_path), + "--enrollments", + str(enrollments_path), + "--output", + str(tls_trust_path), + "--json", + ], + ) + + assert trust_result.exit_code == 0, trust_result.output + assert CheckpointPeerTlsTrust.model_validate_json( + tls_trust_path.read_text(encoding="utf-8") + ).active_enrollment("server-peer") == enrollment + + revocation_template_path = tmp_path / "tls-revocation-template.json" + revocation_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-peer-tls-revocation-template", + "--tls-trust", str(tls_trust_path), + "--peer-trust", str(peer_trust_path), + "--peer-id", "server-peer", "--generation", "1", + "--reason", "key compromise", "--revoked-at", "2026-07-25T00:00:00+00:00", + "--output", str(revocation_template_path), "--json", + ], + ) + assert revocation_result.exit_code == 0, revocation_result.output + revocation_template = CheckpointPeerTlsRevocationTemplate.model_validate_json( + revocation_template_path.read_text(encoding="utf-8") + ) + revocation = build_signed_checkpoint_peer_tls_revocation( + revocation_template, peer_trust, key_id="server-peer-identity-1", + signature_base64=base64.b64encode( + identity_keys["server-peer"].sign(revocation_template.statement.signing_bytes()) + ).decode(), + ) + revocation_path = tmp_path / "tls-revocation.json" + finalize_result = runner.invoke( + app, + [ + "benchmark", "agent-cli-checkpoint-peer-tls-revocation", + "--revocation-template", str(revocation_template_path), + "--peer-trust", str(peer_trust_path), "--key-id", revocation.key_id, + "--signature-base64", revocation.signature_base64, + "--output", str(revocation_path), "--json", + ], + ) + assert finalize_result.exit_code == 0, finalize_result.output + assert CheckpointPeerTlsRevocation.model_validate_json( + revocation_path.read_text(encoding="utf-8") + ) == revocation + serve_help = runner.invoke( + app, + ["benchmark", "agent-cli-checkpoint-gossip-mtls-serve", "--help"], + ) + assert serve_help.exit_code == 0, serve_help.output + assert "Allowed client IP" in serve_help.output + for command in ( + "agent-cli-checkpoint-gossip-mtls-fetch", + "agent-cli-checkpoint-gossip-mtls-ack", + "agent-cli-checkpoint-gossip-mtls-sync", + ): + result = runner.invoke( + app, + ["benchmark", command, "--help"], + color=False, + ) + assert result.exit_code == 0, result.output + help_output = unstyle(result.output) + assert "--tls-trust" in help_output + assert "--private-key" in help_output diff --git a/tests/unit/benchmarks/test_agent_cli_gossip_transport.py b/tests/unit/benchmarks/test_agent_cli_gossip_transport.py new file mode 100644 index 0000000..13ebc31 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_gossip_transport.py @@ -0,0 +1,277 @@ +"""Bounded authenticated loopback gossip transport tests.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import stat +from pathlib import Path + +import pytest + +from benchmarks.agent_cli_gossip_transport import ( + MAX_GOSSIP_REQUEST_BYTES, + MAX_GOSSIP_RESPONSE_BYTES, + CheckpointGossipServer, + send_checkpoint_gossip_request, +) + + +class _RecordingHandler: + def __init__(self, *, delay_seconds: float = 0.0) -> None: + self.delay_seconds = delay_seconds + self.requests: list[tuple[str, dict[str, object]]] = [] + + async def dispatch( + self, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: + if self.delay_seconds: + await asyncio.sleep(self.delay_seconds) + self.requests.append((operation, payload)) + return {"operation": operation, "payload": payload} + + +class _OversizedResponseHandler: + async def dispatch( + self, + operation: str, + payload: dict[str, object], + ) -> dict[str, object]: + return {"blob": "x" * MAX_GOSSIP_RESPONSE_BYTES} + + +@pytest.mark.asyncio +async def test_gossip_transport_authenticates_one_use_nonce_and_removes_descriptor( + tmp_path: Path, +) -> None: + handler = _RecordingHandler() + descriptor_path = tmp_path / "gossip" / "source-peer.json" + nonce = bytes(range(32)) + + async with CheckpointGossipServer( + descriptor_path=descriptor_path, + registry_id="gossip-registry", + source_peer_id="peer-1", + handler=handler, + max_requests=4, + ) as server: + descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) + assert descriptor["host"] == "127.0.0.1" + assert descriptor["protocol_version"] == 1 + assert descriptor["registry_id"] == "gossip-registry" + assert descriptor["source_peer_id"] == "peer-1" + assert descriptor["token"] + assert stat.S_IMODE(descriptor_path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(descriptor_path.stat().st_mode) == 0o600 + + response = await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation="status", + payload={"cursor": 3}, + client_nonce=nonce, + ) + + assert response == { + "operation": "status", + "payload": {"cursor": 3}, + } + assert handler.requests == [("status", {"cursor": 3})] + assert server.completed_requests == 1 + with pytest.raises(RuntimeError, match="replayed_nonce"): + await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation="status", + payload={}, + client_nonce=nonce, + ) + assert handler.requests == [("status", {"cursor": 3})] + + assert descriptor_path.exists() is False + assert server.active_client_count == 0 + + +@pytest.mark.asyncio +async def test_gossip_transport_rejects_wrong_token_version_and_oversized_request( + tmp_path: Path, +) -> None: + handler = _RecordingHandler() + descriptor_path = tmp_path / "gossip.json" + + async with CheckpointGossipServer( + descriptor_path=descriptor_path, + registry_id="gossip-registry", + source_peer_id="peer-1", + handler=handler, + max_requests=4, + ): + original = json.loads(descriptor_path.read_text(encoding="utf-8")) + wrong_token = dict(original) + wrong_token["token"] = base64.b64encode(bytes([9]) * 32).decode() + descriptor_path.write_text(json.dumps(wrong_token), encoding="utf-8") + descriptor_path.chmod(0o600) + with pytest.raises(RuntimeError, match="authentication"): + await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation="status", + payload={}, + ) + + unsupported = dict(original) + unsupported["protocol_version"] = 2 + descriptor_path.write_text(json.dumps(unsupported), encoding="utf-8") + descriptor_path.chmod(0o600) + with pytest.raises(ValueError, match="unsupported.*protocol"): + await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation="status", + payload={}, + ) + + descriptor_path.write_text(json.dumps(original), encoding="utf-8") + descriptor_path.chmod(0o600) + with pytest.raises(ValueError, match="request.*limit"): + await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation="status", + payload={"blob": "x" * MAX_GOSSIP_REQUEST_BYTES}, + ) + + assert handler.requests == [] + + +@pytest.mark.asyncio +async def test_gossip_transport_bounds_dispatch_timeout_and_request_count( + tmp_path: Path, +) -> None: + handler = _RecordingHandler(delay_seconds=0.1) + descriptor_path = tmp_path / "gossip.json" + server = CheckpointGossipServer( + descriptor_path=descriptor_path, + registry_id="gossip-registry", + source_peer_id="peer-1", + handler=handler, + request_timeout_seconds=0.02, + max_requests=1, + ) + await server.start() + + with pytest.raises(RuntimeError, match="request_timeout"): + await send_checkpoint_gossip_request( + descriptor_path=descriptor_path, + operation="status", + payload={}, + request_timeout_seconds=0.5, + ) + await asyncio.wait_for(server.wait_stopped(), timeout=0.5) + await server.close() + + assert server.completed_requests == 1 + assert descriptor_path.exists() is False + assert handler.requests == [] + + +@pytest.mark.asyncio +async def test_gossip_server_closes_stalled_clients_deterministically( + tmp_path: Path, +) -> None: + handler = _RecordingHandler() + descriptor_path = tmp_path / "gossip.json" + server = CheckpointGossipServer( + descriptor_path=descriptor_path, + registry_id="gossip-registry", + source_peer_id="peer-1", + handler=handler, + request_timeout_seconds=10.0, + max_requests=4, + ) + await server.start() + descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) + reader, writer = await asyncio.open_connection( + descriptor["host"], + descriptor["port"], + ) + for _ in range(20): + if server.active_client_count: + break + await asyncio.sleep(0.001) + assert server.active_client_count == 1 + + await asyncio.wait_for(server.close(), timeout=0.5) + + assert await asyncio.wait_for(reader.read(), timeout=0.5) == b"" + assert server.active_client_count == 0 + assert descriptor_path.exists() is False + writer.close() + await writer.wait_closed() + + +@pytest.mark.asyncio +async def test_gossip_server_bounds_read_timeout_concurrency_and_response( + tmp_path: Path, +) -> None: + timeout_descriptor = tmp_path / "timeout.json" + timeout_server = CheckpointGossipServer( + descriptor_path=timeout_descriptor, + registry_id="gossip-registry", + source_peer_id="peer-1", + handler=_RecordingHandler(), + request_timeout_seconds=0.02, + max_requests=2, + ) + await timeout_server.start() + descriptor = json.loads(timeout_descriptor.read_text(encoding="utf-8")) + reader, writer = await asyncio.open_connection( + descriptor["host"], + descriptor["port"], + ) + + timeout_response = json.loads( + await asyncio.wait_for(reader.readline(), timeout=0.5) + ) + + assert timeout_response == {"error": "request_timeout", "ok": False} + writer.close() + await writer.wait_closed() + await timeout_server.close() + + bounded_descriptor = tmp_path / "bounded.json" + bounded_server = CheckpointGossipServer( + descriptor_path=bounded_descriptor, + registry_id="gossip-registry", + source_peer_id="peer-1", + handler=_OversizedResponseHandler(), + max_requests=2, + max_concurrent_clients=1, + ) + await bounded_server.start() + descriptor = json.loads(bounded_descriptor.read_text(encoding="utf-8")) + _, stalled_writer = await asyncio.open_connection( + descriptor["host"], + descriptor["port"], + ) + for _ in range(20): + if bounded_server.active_client_count: + break + await asyncio.sleep(0.001) + with pytest.raises(RuntimeError, match="server_busy"): + await send_checkpoint_gossip_request( + descriptor_path=bounded_descriptor, + operation="status", + payload={}, + ) + stalled_writer.close() + await stalled_writer.wait_closed() + for _ in range(20): + if bounded_server.active_client_count == 0: + break + await asyncio.sleep(0.001) + with pytest.raises(RuntimeError, match="response_too_large"): + await send_checkpoint_gossip_request( + descriptor_path=bounded_descriptor, + operation="status", + payload={}, + ) + await bounded_server.close() diff --git a/tests/unit/benchmarks/test_agent_cli_peer_trust_ledger.py b/tests/unit/benchmarks/test_agent_cli_peer_trust_ledger.py new file mode 100644 index 0000000..9ede40a --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_peer_trust_ledger.py @@ -0,0 +1,336 @@ +"""Peer-trust generation ledger and rollover continuity tests.""" + +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from typer.testing import CliRunner + +from benchmarks.agent_cli_checkpoint_registry import ( + CheckpointPeerKeyDeclaration, + CheckpointPeerTrust, + CheckpointPeerTrustDeclaration, + build_checkpoint_peer_trust, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_peer_trust_ledger import ( + CheckpointPeerRotationSignature, + CheckpointPeerTrustGeneration, + CheckpointPeerTrustLedger, + build_checkpoint_peer_trust_ledger, + build_checkpoint_peer_trust_rotation_certificate, + build_checkpoint_peer_trust_rotation_template, +) +from interface.cli.main import app + +runner = CliRunner() + + +def _private(seed: int) -> Ed25519PrivateKey: + return Ed25519PrivateKey.from_private_bytes(bytes([seed]) * 32) + + +def _trust_context(): + predecessor_keys = { + f"peer-{index}": _private(70 + index) for index in range(1, 4) + } + successor_keys = dict(predecessor_keys) + successor_keys["peer-2-next"] = _private(79) + predecessor = build_checkpoint_peer_trust( + CheckpointPeerTrustDeclaration( + schema_version=SCHEMA_VERSION, + registry_id="rotation-registry", + keys=tuple( + CheckpointPeerKeyDeclaration( + peer_id=peer_id, + key_id=f"{peer_id}-key-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + ) + for peer_id, private_key in predecessor_keys.items() + ), + ) + ) + successor_declarations = [] + for peer_id, private_key in predecessor_keys.items(): + successor_declarations.append( + CheckpointPeerKeyDeclaration( + peer_id=peer_id, + key_id=f"{peer_id}-key-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + status="revoked" if peer_id == "peer-2" else "active", + ) + ) + successor_declarations.append( + CheckpointPeerKeyDeclaration( + peer_id="peer-2", + key_id="peer-2-key-2", + public_key_base64=base64.b64encode( + successor_keys["peer-2-next"].public_key().public_bytes_raw() + ).decode(), + ) + ) + successor = build_checkpoint_peer_trust( + CheckpointPeerTrustDeclaration( + schema_version=SCHEMA_VERSION, + registry_id="rotation-registry", + keys=tuple(successor_declarations), + ) + ) + return predecessor_keys, successor_keys, predecessor, successor + + +def _rotation_certificate( + predecessor: CheckpointPeerTrust, + successor: CheckpointPeerTrust, + predecessor_keys: dict[str, Ed25519PrivateKey], + *, + signers: tuple[str, ...] = ("peer-1", "peer-2"), +): + template = build_checkpoint_peer_trust_rotation_template( + predecessor, + successor, + generation=2, + ) + requests = {request.peer_id: request for request in template.requests} + signatures = tuple( + CheckpointPeerRotationSignature( + peer_id=peer_id, + key_id=f"{peer_id}-key-1", + signature_base64=base64.b64encode( + predecessor_keys[peer_id].sign( + requests[peer_id].statement.signing_bytes() + ) + ).decode(), + ) + for peer_id in signers + ) + return template, build_checkpoint_peer_trust_rotation_certificate( + template, + predecessor, + successor, + signatures, + ) + + +def test_peer_trust_ledger_resolves_historical_and_active_generations() -> None: + predecessor_keys, _, predecessor, successor = _trust_context() + template, rotation = _rotation_certificate( + predecessor, + successor, + predecessor_keys, + ) + + ledger = build_checkpoint_peer_trust_ledger( + ( + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + trust=predecessor, + ), + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=2, + trust=successor, + rotation=rotation, + ), + ) + ) + + assert template.minimum_distinct_peer_signatures == 2 + assert ledger.active_generation == 2 + assert ledger.active_trust == successor + assert ledger.resolve_peer_trust(predecessor.peer_trust_sha256) == predecessor + assert ledger.resolve_peer_trust(successor.peer_trust_sha256) == successor + with pytest.raises(ValueError, match="not present in peer trust ledger"): + ledger.resolve_peer_trust("f" * 64) + + +def test_peer_trust_rotation_rejects_minority_and_untrusted_keys() -> None: + predecessor_keys, successor_keys, predecessor, successor = _trust_context() + template = build_checkpoint_peer_trust_rotation_template( + predecessor, + successor, + generation=2, + ) + peer_1_request = next( + request for request in template.requests if request.peer_id == "peer-1" + ) + minority = ( + CheckpointPeerRotationSignature( + peer_id="peer-1", + key_id="peer-1-key-1", + signature_base64=base64.b64encode( + predecessor_keys["peer-1"].sign( + peer_1_request.statement.signing_bytes() + ) + ).decode(), + ), + ) + + with pytest.raises(ValueError, match="strict-majority quorum is incomplete"): + build_checkpoint_peer_trust_rotation_certificate( + template, + predecessor, + successor, + minority, + ) + + peer_2_request = next( + request for request in template.requests if request.peer_id == "peer-2" + ) + unknown_rotation_key = minority + ( + CheckpointPeerRotationSignature( + peer_id="peer-2", + key_id="peer-2-key-2", + signature_base64=base64.b64encode( + successor_keys["peer-2-next"].sign( + peer_2_request.statement.signing_bytes() + ) + ).decode(), + ), + ) + with pytest.raises(ValueError, match="active predecessor key"): + build_checkpoint_peer_trust_rotation_certificate( + template, + predecessor, + successor, + unknown_rotation_key, + ) + + invalid_signature = minority + ( + CheckpointPeerRotationSignature( + peer_id="peer-2", + key_id="peer-2-key-1", + signature_base64=base64.b64encode(bytes(64)).decode(), + ), + ) + with pytest.raises(ValueError, match="signature is invalid: peer-2"): + build_checkpoint_peer_trust_rotation_certificate( + template, + predecessor, + successor, + invalid_signature, + ) + + +def test_peer_trust_ledger_rejects_gaps_reuse_and_tampering() -> None: + predecessor_keys, _, predecessor, successor = _trust_context() + _, rotation = _rotation_certificate( + predecessor, + successor, + predecessor_keys, + ) + genesis = CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + trust=predecessor, + ) + successor_generation = CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=2, + trust=successor, + rotation=rotation, + ) + ledger = build_checkpoint_peer_trust_ledger((genesis, successor_generation)) + + with pytest.raises(ValueError, match="contiguous"): + build_checkpoint_peer_trust_ledger( + ( + genesis, + successor_generation.model_copy(update={"generation": 3}), + ) + ) + with pytest.raises(ValueError, match="must change peer trust"): + build_checkpoint_peer_trust_rotation_template( + predecessor, + predecessor, + generation=2, + ) + payload = ledger.model_dump(mode="json") + payload["ledger_sha256"] = "f" * 64 + with pytest.raises(ValueError, match="fingerprint"): + CheckpointPeerTrustLedger.model_validate(payload) + + +def test_peer_trust_rotation_and_ledger_cli_are_private_key_free( + tmp_path: Path, +) -> None: + predecessor_keys, _, predecessor, successor = _trust_context() + predecessor_path = tmp_path / "predecessor.json" + successor_path = tmp_path / "successor.json" + predecessor_path.write_text(predecessor.to_json(), encoding="utf-8") + successor_path.write_text(successor.to_json(), encoding="utf-8") + template_path = tmp_path / "rotation-template.json" + + template_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-peer-trust-rotation-template", + "--predecessor-peer-trust", + str(predecessor_path), + "--successor-peer-trust", + str(successor_path), + "--generation", + "2", + "--output", + str(template_path), + "--json", + ], + ) + assert template_result.exit_code == 0, template_result.output + assert "private" not in template_path.read_text(encoding="utf-8").lower() + _, rotation = _rotation_certificate( + predecessor, + successor, + predecessor_keys, + ) + generations_path = tmp_path / "trust-generations.json" + generations_path.write_text( + json.dumps( + { + "generations": [ + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + trust=predecessor, + ).model_dump(mode="json"), + CheckpointPeerTrustGeneration( + schema_version=SCHEMA_VERSION, + generation=2, + trust=successor, + rotation=rotation, + ).model_dump(mode="json"), + ] + }, + sort_keys=True, + ), + encoding="utf-8", + ) + ledger_path = tmp_path / "peer-trust-ledger.json" + ledger_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-checkpoint-peer-trust-ledger", + "--generations", + str(generations_path), + "--output", + str(ledger_path), + "--json", + ], + ) + + assert ledger_result.exit_code == 0, ledger_result.output + ledger = CheckpointPeerTrustLedger.model_validate_json( + ledger_path.read_text(encoding="utf-8") + ) + assert ledger.active_trust == successor diff --git a/tests/unit/benchmarks/test_agent_cli_preflight.py b/tests/unit/benchmarks/test_agent_cli_preflight.py new file mode 100644 index 0000000..d480d93 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_preflight.py @@ -0,0 +1,2013 @@ +"""Tests for agent CLI campaign preflight and review template binding.""" + +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from pydantic import ValidationError +from typer.testing import CliRunner + +from benchmarks.agent_cli_adjudication import ( + AdjudicationReviews, + RecordedEvidence, + finalize_recorded_results, +) +from benchmarks.agent_cli_attestation import ( + ReviewAttestationBundle, + ReviewerPublicKeyDeclaration, + ReviewerTrustDeclaration, + SignedReviewAttestation, + build_review_attestation_template, + build_reviewer_trust, + verify_review_attestations, +) +from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + ReviewerEnrollmentCertificate, + SignedCampaignEnvelope, + build_benchmark_authority, + build_campaign_envelope_request, + build_reviewer_enrollment_bundle, + build_reviewer_enrollment_statement, + build_reviewer_enrollment_template, + verify_reviewer_enrollments, + verify_signed_campaign_envelope, +) +from benchmarks.agent_cli_campaign import CampaignStage, build_campaign_status +from benchmarks.agent_cli_comparison import AgentCliArm, AgentCliManifest +from benchmarks.agent_cli_preflight import ( + CampaignPreflight, + RuntimeVersionBundle, + build_campaign_preflight, + build_review_template, + recorded_evidence_sha256, + validate_review_bindings, +) +from benchmarks.agent_cli_recorder import AgentCliRecorderConfig +from benchmarks.agent_cli_review_policy import ( + ReviewerPolicyDeclaration, + build_reviewer_policy, + validate_reviewer_policy_capacity, + validate_reviewer_separation, +) +from benchmarks.agent_cli_transparency import ( + AuthorityRootGeneration, + AuthorityRotationCertificate, + SignedAuthorityRootLedger, + SignedTransparencyTreeHead, + TransparencyLogEntry, + build_authority_root_ledger_request, + build_authority_rotation_request, + build_transparency_consistency_proof, + build_transparency_inclusion_proof, + build_transparency_log, + build_transparency_tree_head_request, +) +from benchmarks.agent_cli_witness import ( + TransparencyWitnessKeyDeclaration, + TransparencyWitnessSignature, + TransparencyWitnessTrustDeclaration, + build_transparency_witness_trust, + build_witness_checkpoint_bundle, + build_witness_checkpoint_template, +) +from interface.cli.main import app + +runner = CliRunner() +REVISION = "a" * 40 + + +def _manifest() -> AgentCliManifest: + return AgentCliManifest.model_validate( + { + "schema_version": 1, + "benchmark_id": "campaign-001", + "task": { + "id": "task-001", + "goal": "Implement the selected task.", + "workspace_revision": REVISION, + "checks": ["unit"], + "handoff_assertions": ["decision"], + }, + "arms": ["codex_cli", "claude_code", "morphic_control"], + "repetitions": 1, + } + ) + + +def _config() -> AgentCliRecorderConfig: + return AgentCliRecorderConfig.model_validate( + { + "schema_version": 1, + "benchmark_id": "campaign-001", + "arm_commands": { + "codex_cli": ["codex", "exec", "{goal}"], + "claude_code": ["claude", "-p", "{goal}"], + "morphic_control": [ + "morphic", + "code", + "--benchmark-receipt", + "{goal}", + ], + }, + "check_commands": {"unit": ["python", "-m", "pytest", "-q"]}, + "handoff_commands": {"decision": ["python", "-c", "raise SystemExit(0)"]}, + "estimated_cost_usd_per_trial": { + "codex_cli": 0.5, + "claude_code": 0.5, + "morphic_control": 1.0, + }, + "model_hints": {"codex_cli": "o4-mini"}, + "timeout_seconds": 300.0, + } + ) + + +def _versions() -> RuntimeVersionBundle: + return RuntimeVersionBundle.model_validate( + { + "schema_version": 1, + "benchmark_id": "campaign-001", + "runtimes": { + "codex_cli": {"executable": "codex", "version": "codex-cli 1.2.3"}, + "claude_code": {"executable": "claude", "version": "claude 2.3.4"}, + "morphic_control": { + "executable": "morphic", + "version": "morphic-agent 0.6.3", + }, + }, + } + ) + + +def _command(seed: str) -> dict[str, object]: + return { + "argv_sha256": seed * 64, + "exit_code": 0, + "timed_out": False, + "elapsed_seconds": 1.0, + "stdout_sha256": "b" * 64, + "stdout_bytes": 10, + "stderr_sha256": "c" * 64, + "stderr_bytes": 0, + } + + +def _evidence() -> RecordedEvidence: + receipts = { + "codex_cli": { + "provider": "codex_cli", + "success": True, + "model": "o4-mini", + "usage": {"prompt_tokens": 100, "completion_tokens": 20}, + "cost_usd": 0.000198, + "cost_source": "calculated_from_usage", + "parse_errors": 0, + }, + "claude_code": { + "provider": "claude_code", + "success": True, + "model": "claude-sonnet", + "usage": {"input_tokens": 100, "output_tokens": 10}, + "cost_usd": 0.01, + "cost_source": "provider_reported", + "parse_errors": 0, + }, + "morphic_control": { + "provider": "morphic_control", + "success": True, + "model": "morphic-control[codex_cli]", + "usage": {"input_tokens": 100, "output_tokens": 10}, + "cost_usd": 0.02, + "cost_source": "morphic_reported", + "parse_errors": 0, + }, + } + trials = [] + for index, arm in enumerate(AgentCliArm, start=1): + trials.append( + { + "arm": arm.value, + "trial": 1, + "reserved_cost_usd": 1.0, + "agent": _command(str(index)), + "checks": {"unit": _command("d")}, + "handoff_assertions": {"decision": _command("e")}, + "receipt": receipts[arm.value], + "completed": True, + "passed_checks": ["unit"], + "passed_handoff_assertions": ["decision"], + } + ) + return RecordedEvidence.model_validate( + { + "schema_version": 1, + "benchmark_id": "campaign-001", + "task_id": "task-001", + "workspace_revision": REVISION, + "estimated_max_cost_usd": 2.0, + "authorized_cost_cap_usd": 2.0, + "cost_collection": "normalized_receipts", + "trials": trials, + } + ) + + +def _preflight(): + return build_campaign_preflight( + _manifest(), + _config(), + _versions(), + resolved_revision=REVISION, + ) + + +def _policy(): + return build_reviewer_policy( + ReviewerPolicyDeclaration.model_validate( + { + "schema_version": 1, + "benchmark_id": "campaign-001", + "operator_id": "operator-1", + "reviewer_ids": ["reviewer-2", "reviewer-1"], + "minimum_distinct_reviewers": 2, + } + ) + ) + + +def _completed_reviews( + *, + reviewer_ids: tuple[str, str, str] | None = None, + reviewer_trust_sha256: str | None = None, +): + evidence = _evidence() + policy = _policy() + payload = build_review_template( + _preflight(), + evidence, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=reviewer_trust_sha256, + ).model_dump(mode="json") + payload["review_completed"] = True + assigned = reviewer_ids or ("reviewer-1", "reviewer-2", "reviewer-1") + for decision, reviewer_id in zip(payload["decisions"], assigned, strict=True): + decision.update( + accepted_patch=False, + human_interventions=0, + recovery_attempted=False, + recovery_succeeded=False, + reviewer_id=reviewer_id, + review_artifact_sha256="f" * 64, + ) + return AdjudicationReviews.model_validate(payload) + + +def _private_keys() -> dict[str, Ed25519PrivateKey]: + return { + "reviewer-1": Ed25519PrivateKey.from_private_bytes(b"\x01" * 32), + "reviewer-2": Ed25519PrivateKey.from_private_bytes(b"\x02" * 32), + } + + +def _trust_declaration( + *, + revoke_reviewer_2: bool = False, + reviewer_authority_sha256: str | None = None, + authority_root_ledger_sha256: str | None = None, +) -> ReviewerTrustDeclaration: + keys = _private_keys() + declarations = [] + for reviewer_id, private_key in keys.items(): + public_bytes = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + declarations.append( + ReviewerPublicKeyDeclaration( + reviewer_id=reviewer_id, + key_id=f"{reviewer_id}-key-1", + public_key_base64=base64.b64encode(public_bytes).decode(), + status=( + "revoked" + if revoke_reviewer_2 and reviewer_id == "reviewer-2" + else "active" + ), + ) + ) + return ReviewerTrustDeclaration( + schema_version=1, + benchmark_id="campaign-001", + review_policy_sha256=_policy().policy_sha256, + reviewer_authority_sha256=reviewer_authority_sha256, + authority_root_ledger_sha256=authority_root_ledger_sha256, + keys=tuple(reversed(declarations)), + ) + + +def _review_attestation_bundle(policy, trust, reviews): + template = build_review_attestation_template(policy, trust, reviews) + private_keys = _private_keys() + signed = tuple( + SignedReviewAttestation( + statement=request.statement, + key_id=f"{request.statement.reviewer_id}-key-1", + signature_base64=base64.b64encode( + private_keys[request.statement.reviewer_id].sign( + request.statement.signing_bytes() + ) + ).decode(), + ) + for request in template.requests + ) + return ReviewAttestationBundle( + schema_version=1, + benchmark_id=reviews.benchmark_id, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=trust.reviewer_trust_sha256, + reviews_sha256=template.reviews_sha256, + attestations=signed, + ) + + +def _signed_attestations(): + policy = _policy() + trust = build_reviewer_trust(_trust_declaration(), policy) + reviews = _completed_reviews(reviewer_trust_sha256=trust.reviewer_trust_sha256) + return policy, trust, reviews, _review_attestation_bundle(policy, trust, reviews) + + +def _authority_private_key() -> Ed25519PrivateKey: + return Ed25519PrivateKey.from_private_bytes(b"\x03" * 32) + + +def _authority_declaration() -> BenchmarkAuthorityDeclaration: + public_bytes = _authority_private_key().public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + return BenchmarkAuthorityDeclaration( + schema_version=1, + authority_id="example-org-benchmark-ca", + public_key_base64=base64.b64encode(public_bytes).decode(), + ) + + +def _anchored_artifacts(): + policy = _policy() + authority = build_benchmark_authority(_authority_declaration()) + trust = build_reviewer_trust( + _trust_declaration(reviewer_authority_sha256=authority.authority_sha256), + policy, + ) + reviews = _completed_reviews(reviewer_trust_sha256=trust.reviewer_trust_sha256) + attestations = _review_attestation_bundle(policy, trust, reviews) + certificates = [] + for key in trust.keys: + statement = build_reviewer_enrollment_statement( + authority, + policy, + trust, + key, + ) + certificates.append( + ReviewerEnrollmentCertificate( + statement=statement, + signature_base64=base64.b64encode( + _authority_private_key().sign(statement.signing_bytes()) + ).decode(), + ) + ) + enrollments = build_reviewer_enrollment_bundle( + authority, + policy, + trust, + tuple(reversed(certificates)), + ) + results = finalize_recorded_results( + _manifest(), + _evidence(), + reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + reviewer_authority=authority, + reviewer_enrollments=enrollments, + ) + request = build_campaign_envelope_request( + authority=authority, + manifest=_manifest(), + preflight=_preflight(), + evidence=_evidence(), + reviews=reviews, + review_policy=policy, + reviewer_trust=trust, + reviewer_enrollments=enrollments, + attestations=attestations, + results=results, + ) + envelope = SignedCampaignEnvelope( + statement=request.statement, + signature_base64=base64.b64encode( + _authority_private_key().sign(request.statement.signing_bytes()) + ).decode(), + ) + return ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + results, + request, + envelope, + ) + + +def _ledger_bound_artifacts(): + genesis_private = Ed25519PrivateKey.from_private_bytes(b"\x04" * 32) + genesis_public = genesis_private.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + genesis = build_benchmark_authority( + BenchmarkAuthorityDeclaration( + schema_version=1, + authority_id="example-org-benchmark-ca-genesis", + public_key_base64=base64.b64encode(genesis_public).decode(), + ) + ) + authority = build_benchmark_authority(_authority_declaration()) + rotation_request = build_authority_rotation_request( + generation=2, + predecessor=genesis, + successor=authority, + ) + rotation = AuthorityRotationCertificate( + statement=rotation_request.statement, + signature_base64=base64.b64encode( + genesis_private.sign(rotation_request.statement.signing_bytes()) + ).decode(), + ) + ledger_request = build_authority_root_ledger_request( + ( + AuthorityRootGeneration( + schema_version=1, + generation=1, + authority=genesis, + ), + AuthorityRootGeneration( + schema_version=1, + generation=2, + authority=authority, + rotation=rotation, + ), + ), + revoked_authority_sha256=(genesis.authority_sha256,), + ) + ledger = SignedAuthorityRootLedger( + statement=ledger_request.statement, + signature_base64=base64.b64encode( + _authority_private_key().sign(ledger_request.statement.signing_bytes()) + ).decode(), + ) + policy = _policy() + trust = build_reviewer_trust( + _trust_declaration( + reviewer_authority_sha256=authority.authority_sha256, + authority_root_ledger_sha256=ledger.statement.ledger_sha256, + ), + policy, + ) + reviews = _completed_reviews(reviewer_trust_sha256=trust.reviewer_trust_sha256) + attestations = _review_attestation_bundle(policy, trust, reviews) + certificates = [] + for key in trust.keys: + statement = build_reviewer_enrollment_statement(authority, policy, trust, key) + certificates.append( + ReviewerEnrollmentCertificate( + statement=statement, + signature_base64=base64.b64encode( + _authority_private_key().sign(statement.signing_bytes()) + ).decode(), + ) + ) + enrollments = build_reviewer_enrollment_bundle( + authority, + policy, + trust, + tuple(certificates), + ) + results = finalize_recorded_results( + _manifest(), + _evidence(), + reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + reviewer_authority=authority, + reviewer_enrollments=enrollments, + authority_root_ledger=ledger, + ) + envelope_request = build_campaign_envelope_request( + authority=authority, + manifest=_manifest(), + preflight=_preflight(), + evidence=_evidence(), + reviews=reviews, + review_policy=policy, + reviewer_trust=trust, + reviewer_enrollments=enrollments, + attestations=attestations, + results=results, + authority_root_ledger=ledger, + ) + envelope = SignedCampaignEnvelope( + statement=envelope_request.statement, + signature_base64=base64.b64encode( + _authority_private_key().sign(envelope_request.statement.signing_bytes()) + ).decode(), + ) + log = build_transparency_log( + "example-org-agent-cli", + ( + TransparencyLogEntry( + sequence=0, + kind="authority_root_ledger", + artifact_sha256=ledger.statement.ledger_sha256, + ), + TransparencyLogEntry( + sequence=1, + kind="reviewer_enrollments", + artifact_sha256=enrollments.reviewer_enrollments_sha256, + ), + TransparencyLogEntry( + sequence=2, + kind="campaign_envelope", + artifact_sha256=envelope.statement.envelope_sha256, + ), + ), + ) + head_request = build_transparency_tree_head_request(log, ledger) + head = SignedTransparencyTreeHead( + statement=head_request.statement, + signature_base64=base64.b64encode( + _authority_private_key().sign(head_request.statement.signing_bytes()) + ).decode(), + ) + proof = build_transparency_inclusion_proof(log, leaf_index=2, tree_head=head) + return ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + results, + ledger, + envelope, + proof, + log, + ) + + +def test_preflight_is_deterministic_non_authorizing_and_prompt_free() -> None: + first = _preflight() + second = _preflight() + + assert first.to_json() == second.to_json() + assert first.execution_authorized is False + assert first.estimated_max_cost_usd == 2.0 + assert first.trial_count == 3 + assert len(first.preflight_sha256) == 64 + assert len(first.manifest_sha256) == 64 + assert len(first.config_sha256) == 64 + assert "Implement the selected task" not in first.to_json() + assert set(first.runtime_fingerprints) == set(AgentCliArm) + + +def test_preflight_binds_manifest_goal_without_exposing_it() -> None: + changed_payload = _manifest().model_dump(mode="json") + changed_payload["task"]["goal"] = "A different selected task." + changed = build_campaign_preflight( + AgentCliManifest.model_validate(changed_payload), + _config(), + _versions(), + resolved_revision=REVISION, + ) + + assert changed.manifest_sha256 != _preflight().manifest_sha256 + assert "A different selected task" not in changed.to_json() + + +def test_preflight_normalizes_and_fingerprints_runtime_versions() -> None: + payload = _versions().model_dump(mode="json") + payload["runtimes"]["codex_cli"]["version"] = " codex-cli\n1.2.3 " + + report = build_campaign_preflight( + _manifest(), + _config(), + RuntimeVersionBundle.model_validate(payload), + resolved_revision=REVISION, + ) + + runtime = report.runtime_fingerprints[AgentCliArm.CODEX_CLI] + assert runtime.version == "codex-cli 1.2.3" + assert len(runtime.version_sha256) == 64 + + +@pytest.mark.parametrize( + ("mutation", "match"), + [ + (lambda payload: payload["runtimes"].pop("claude_code"), "exactly"), + ( + lambda payload: payload["runtimes"]["codex_cli"].update( + executable="different" + ), + "executable", + ), + ], +) +def test_preflight_rejects_incomplete_or_mismatched_runtime_declarations( + mutation: object, + match: str, +) -> None: + payload = _versions().model_dump(mode="json") + mutation(payload) # type: ignore[operator] + + with pytest.raises((ValidationError, ValueError), match=match): + build_campaign_preflight( + _manifest(), + _config(), + RuntimeVersionBundle.model_validate(payload), + resolved_revision=REVISION, + ) + + +def test_preflight_requires_manifest_to_pin_full_resolved_commit() -> None: + with pytest.raises(ValueError, match="immutable resolved revision"): + build_campaign_preflight( + _manifest(), + _config(), + _versions(), + resolved_revision="b" * 40, + ) + + +def test_preflight_report_rejects_artifact_tampering() -> None: + payload = _preflight().model_dump(mode="json") + payload["estimated_max_cost_usd"] = 1.0 + + with pytest.raises(ValidationError, match="preflight fingerprint"): + CampaignPreflight.model_validate(payload) + + +def test_review_template_binds_every_trial_to_preflight_and_evidence() -> None: + evidence = _evidence() + template = build_review_template(_preflight(), evidence) + payload = template.model_dump(mode="json") + + assert template.preflight_sha256 == _preflight().preflight_sha256 + assert template.evidence_sha256 == recorded_evidence_sha256(evidence) + assert len(template.decisions) == 3 + assert payload["decisions"][0]["accepted_patch"] is None + assert payload["decisions"][0]["reviewer_id"] is None + assert payload["review_completed"] is False + + +def test_completed_review_bindings_validate_and_finalize() -> None: + evidence = _evidence() + preflight = _preflight() + template = build_review_template(preflight, evidence).model_dump(mode="json") + template["review_completed"] = True + for decision in template["decisions"]: + decision.update( + accepted_patch=True, + human_interventions=0, + recovery_attempted=False, + recovery_succeeded=False, + reviewer_id="reviewer-1", + review_artifact_sha256="f" * 64, + ) + reviews = AdjudicationReviews.model_validate(template) + + validate_review_bindings(preflight, evidence, reviews) + results = finalize_recorded_results(_manifest(), evidence, reviews) + + assert len(results.observations) == 3 + assert all(observation.accepted_patch for observation in results.observations) + + +def test_finalizer_rejects_review_bound_to_different_evidence() -> None: + evidence = _evidence() + payload = build_review_template(_preflight(), evidence).model_dump(mode="json") + payload["review_completed"] = True + for decision in payload["decisions"]: + decision.update( + accepted_patch=False, + human_interventions=0, + recovery_attempted=False, + recovery_succeeded=False, + reviewer_id="reviewer-1", + review_artifact_sha256="f" * 64, + ) + reviews = AdjudicationReviews.model_validate(payload) + tampered = evidence.model_copy(update={"authorized_cost_cap_usd": 1.5}) + + with pytest.raises(ValueError, match="evidence fingerprint"): + finalize_recorded_results(_manifest(), tampered, reviews) + + +def test_committed_runtime_version_template_validates() -> None: + root = Path(__file__).parents[3] + payload = json.loads( + (root / "benchmarks/templates/agent_cli_runtime_versions.example.json").read_text() + ) + + bundle = RuntimeVersionBundle.model_validate(payload) + + assert bundle.benchmark_id == "agent-cli-local-rehearsal" + assert set(bundle.runtimes) == set(AgentCliArm) + + +def test_preflight_cli_publishes_exclusively( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from benchmarks import agent_cli_preflight + + async def resolved(*_args: object, **_kwargs: object) -> str: + return REVISION + + monkeypatch.setattr(agent_cli_preflight, "resolve_git_revision", resolved) + manifest = tmp_path / "manifest.json" + config = tmp_path / "config.json" + versions = tmp_path / "versions.json" + output = tmp_path / "preflight.json" + manifest.write_text(_manifest().model_dump_json(), encoding="utf-8") + config.write_text(_config().model_dump_json(), encoding="utf-8") + versions.write_text(_versions().model_dump_json(), encoding="utf-8") + + args = [ + "benchmark", + "agent-cli-preflight", + "--manifest", + str(manifest), + "--config", + str(config), + "--runtime-versions", + str(versions), + "--source-root", + str(tmp_path), + "--output", + str(output), + "--json", + ] + first = runner.invoke(app, args) + second = runner.invoke(app, args) + + assert first.exit_code == 0 + assert json.loads(first.output)["execution_authorized"] is False + assert second.exit_code == 1 + assert "already exists" in second.output + + +def test_review_template_cli_writes_null_decisions(tmp_path: Path) -> None: + preflight = tmp_path / "preflight.json" + evidence = tmp_path / "evidence.json" + output = tmp_path / "reviews.json" + preflight.write_text(_preflight().to_json(), encoding="utf-8") + evidence.write_text( + json.dumps(_evidence().model_dump(mode="json"), sort_keys=True), + encoding="utf-8", + ) + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-review-template", + "--preflight", + str(preflight), + "--evidence", + str(evidence), + "--output", + str(output), + "--json", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["review_completed"] is False + assert all(row["accepted_patch"] is None for row in payload["decisions"]) + + +def test_finalize_cli_requires_preflight_for_bound_reviews(tmp_path: Path) -> None: + manifest_path = tmp_path / "manifest.json" + preflight_path = tmp_path / "preflight.json" + evidence_path = tmp_path / "evidence.json" + reviews_path = tmp_path / "reviews.json" + output_path = tmp_path / "results.json" + evidence = _evidence() + preflight = _preflight() + payload = build_review_template(preflight, evidence).model_dump(mode="json") + payload["review_completed"] = True + for decision in payload["decisions"]: + decision.update( + accepted_patch=False, + human_interventions=0, + recovery_attempted=False, + recovery_succeeded=False, + reviewer_id="reviewer-1", + review_artifact_sha256="f" * 64, + ) + manifest_path.write_text(_manifest().model_dump_json(), encoding="utf-8") + preflight_path.write_text(preflight.to_json(), encoding="utf-8") + evidence_path.write_text( + json.dumps(evidence.model_dump(mode="json"), sort_keys=True), + encoding="utf-8", + ) + reviews_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + base_args = [ + "benchmark", + "agent-cli-finalize", + "--manifest", + str(manifest_path), + "--evidence", + str(evidence_path), + "--reviews", + str(reviews_path), + "--output", + str(output_path), + ] + + refused = runner.invoke(app, base_args) + accepted = runner.invoke( + app, + [*base_args, "--preflight", str(preflight_path)], + ) + + assert refused.exit_code == 1 + assert "--preflight is required" in refused.output + assert accepted.exit_code == 0 + assert output_path.exists() + + +def test_reviewer_policy_is_deterministic_and_normalizes_reviewer_order() -> None: + first = _policy() + second = _policy() + + assert first.to_json() == second.to_json() + assert first.reviewer_ids == ("reviewer-1", "reviewer-2") + assert len(first.policy_sha256) == 64 + + +@pytest.mark.parametrize( + ("payload", "match"), + [ + ( + { + "operator_id": "same", + "reviewer_ids": ["same"], + "minimum_distinct_reviewers": 1, + }, + "operator", + ), + ( + { + "operator_id": "operator", + "reviewer_ids": ["reviewer", "reviewer"], + "minimum_distinct_reviewers": 1, + }, + "unique", + ), + ( + { + "operator_id": "operator", + "reviewer_ids": ["reviewer"], + "minimum_distinct_reviewers": 2, + }, + "minimum", + ), + ], +) +def test_reviewer_policy_rejects_invalid_separation(payload: dict, match: str) -> None: + with pytest.raises((ValidationError, ValueError), match=match): + build_reviewer_policy( + ReviewerPolicyDeclaration.model_validate( + {"schema_version": 1, "benchmark_id": "campaign-001", **payload} + ) + ) + + +def test_review_template_binds_reviewer_policy() -> None: + template = build_review_template( + _preflight(), + _evidence(), + review_policy_sha256=_policy().policy_sha256, + ) + + assert template.review_policy_sha256 == _policy().policy_sha256 + + +def test_reviewer_policy_rejects_impossible_campaign_capacity() -> None: + declaration = ReviewerPolicyDeclaration.model_validate( + { + "schema_version": 1, + "benchmark_id": "campaign-001", + "operator_id": "operator", + "reviewer_ids": ["r1", "r2", "r3", "r4"], + "minimum_distinct_reviewers": 4, + } + ) + + with pytest.raises(ValueError, match="campaign decision count"): + validate_reviewer_policy_capacity( + build_reviewer_policy(declaration), + decision_count=3, + ) + + +def test_reviewer_separation_accepts_allowed_distinct_reviewers() -> None: + validate_reviewer_separation(_policy(), _completed_reviews()) + + +@pytest.mark.parametrize( + ("reviewer_ids", "match"), + [ + (("operator-1", "reviewer-2", "reviewer-1"), "operator"), + (("outsider", "reviewer-2", "reviewer-1"), "not allowed"), + (("reviewer-1", "reviewer-1", "reviewer-1"), "distinct"), + ], +) +def test_reviewer_separation_rejects_policy_violations( + reviewer_ids: tuple[str, str, str], + match: str, +) -> None: + with pytest.raises(ValueError, match=match): + validate_reviewer_separation( + _policy(), + _completed_reviews(reviewer_ids=reviewer_ids), + ) + + +def test_campaign_status_advances_deterministically_without_authorization() -> None: + manifest = _manifest() + preflight = _preflight() + evidence = _evidence() + policy, trust, reviews, attestations = _signed_attestations() + template = build_review_template( + preflight, + evidence, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=trust.reviewer_trust_sha256, + ) + results = finalize_recorded_results( + manifest, + evidence, + reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + ) + + statuses = [ + build_campaign_status(manifest), + build_campaign_status(manifest, preflight=preflight), + build_campaign_status(manifest, preflight=preflight, evidence=evidence), + build_campaign_status( + manifest, + preflight=preflight, + evidence=evidence, + review_template=template, + review_policy=policy, + reviewer_trust=trust, + ), + build_campaign_status( + manifest, + preflight=preflight, + evidence=evidence, + reviews=reviews, + review_policy=policy, + reviewer_trust=trust, + ), + build_campaign_status( + manifest, + preflight=preflight, + evidence=evidence, + reviews=reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + ), + build_campaign_status( + manifest, + preflight=preflight, + evidence=evidence, + reviews=reviews, + results=results, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + ), + ] + + assert [status.stage for status in statuses] == [ + CampaignStage.MANIFEST_READY, + CampaignStage.PREFLIGHT_READY, + CampaignStage.RECORDED, + CampaignStage.REVIEW_PENDING, + CampaignStage.REVIEW_ATTESTATION_PENDING, + CampaignStage.REVIEW_COMPLETE, + CampaignStage.FINALIZED, + ] + assert all(status.paid_execution_authorized is False for status in statuses) + assert statuses[-2].attestations_verified is True + assert statuses[-1].next_action == "campaign_complete" + assert statuses[-1].to_json() == build_campaign_status( + manifest, + preflight=preflight, + evidence=evidence, + reviews=reviews, + results=results, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + ).to_json() + + +def test_campaign_status_rejects_artifact_order_gaps() -> None: + with pytest.raises(ValueError, match="preflight"): + build_campaign_status(_manifest(), evidence=_evidence()) + + +def test_campaign_status_rejects_manifest_contract_mismatch() -> None: + changed_payload = _manifest().model_dump(mode="json") + changed_payload["task"]["goal"] = "Different goal under the same identity." + changed_preflight = build_campaign_preflight( + AgentCliManifest.model_validate(changed_payload), + _config(), + _versions(), + resolved_revision=REVISION, + ) + + with pytest.raises(ValueError, match="manifest fingerprint"): + build_campaign_status(_manifest(), preflight=changed_preflight) + + +def test_campaign_status_rejects_evidence_estimate_mismatch() -> None: + evidence = _evidence().model_copy(update={"estimated_max_cost_usd": 1.5}) + + with pytest.raises(ValueError, match="evidence estimate"): + build_campaign_status(_manifest(), preflight=_preflight(), evidence=evidence) + + +def test_campaign_status_cli_is_read_only(tmp_path: Path) -> None: + manifest = tmp_path / "manifest.json" + preflight = tmp_path / "preflight.json" + evidence = tmp_path / "evidence.json" + reviews = tmp_path / "reviews.json" + policy = tmp_path / "policy.json" + manifest.write_text(_manifest().model_dump_json(), encoding="utf-8") + preflight.write_text(_preflight().to_json(), encoding="utf-8") + evidence.write_text( + json.dumps(_evidence().model_dump(mode="json"), sort_keys=True), + encoding="utf-8", + ) + reviews.write_text(_completed_reviews().model_dump_json(), encoding="utf-8") + policy.write_text( + json.dumps( + { + "schema_version": 1, + "benchmark_id": "campaign-001", + "operator_id": "operator-1", + "reviewer_ids": ["reviewer-1", "reviewer-2"], + "minimum_distinct_reviewers": 2, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + before = {path: path.read_bytes() for path in tmp_path.iterdir()} + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-status", + "--manifest", + str(manifest), + "--preflight", + str(preflight), + "--evidence", + str(evidence), + "--reviews", + str(reviews), + "--review-policy", + str(policy), + "--json", + ], + ) + + after = {path: path.read_bytes() for path in tmp_path.iterdir()} + assert result.exit_code == 0 + assert json.loads(result.output)["stage"] == "review_complete" + assert before == after + + +def test_committed_review_policy_template_validates() -> None: + root = Path(__file__).parents[3] + payload = json.loads( + (root / "benchmarks/templates/agent_cli_review_policy.example.json").read_text() + ) + + declaration = ReviewerPolicyDeclaration.model_validate(payload) + + assert build_reviewer_policy(declaration).benchmark_id == "agent-cli-local-rehearsal" + + +def test_reviewer_trust_is_deterministic_and_normalizes_key_order() -> None: + first = build_reviewer_trust(_trust_declaration(), _policy()) + second = build_reviewer_trust(_trust_declaration(), _policy()) + + assert first.to_json() == second.to_json() + assert [key.reviewer_id for key in first.keys] == ["reviewer-1", "reviewer-2"] + assert len(first.reviewer_trust_sha256) == 64 + assert all(len(key.public_key_sha256) == 64 for key in first.keys) + + +def test_reviewer_trust_rejects_unauthorized_or_missing_active_keys() -> None: + payload = _trust_declaration().model_dump(mode="json") + payload["keys"][0]["reviewer_id"] = "outsider" + with pytest.raises(ValueError, match="allowed reviewer"): + build_reviewer_trust(ReviewerTrustDeclaration.model_validate(payload), _policy()) + + with pytest.raises(ValueError, match="active key"): + build_reviewer_trust(_trust_declaration(revoke_reviewer_2=True), _policy()) + + +def test_attestation_template_binds_exact_review_provenance_without_private_keys() -> None: + policy, trust, reviews, _bundle = _signed_attestations() + + template = build_review_attestation_template(policy, trust, reviews) + + assert template.reviewer_trust_sha256 == trust.reviewer_trust_sha256 + assert len(template.requests) == 2 + assert {request.statement.reviewer_id for request in template.requests} == { + "reviewer-1", + "reviewer-2", + } + assert all( + request.statement.reviews_sha256 == template.reviews_sha256 + for request in template.requests + ) + assert "private" not in template.to_json().lower() + + +def test_attestation_verifier_accepts_one_active_signature_per_reviewer() -> None: + policy, trust, reviews, bundle = _signed_attestations() + + verify_review_attestations(policy, trust, reviews, bundle) + + +@pytest.mark.parametrize("mutation", ["signature", "missing", "mixed_reviews"]) +def test_attestation_verifier_rejects_tampering_and_incomplete_bundles( + mutation: str, +) -> None: + policy, trust, reviews, bundle = _signed_attestations() + payload = bundle.model_dump(mode="json") + if mutation == "signature": + payload["attestations"][0]["signature_base64"] = base64.b64encode(b"x" * 64).decode() + elif mutation == "missing": + payload["attestations"].pop() + else: + payload["reviews_sha256"] = "f" * 64 + + with pytest.raises(ValueError, match="signature|attestation|reviews fingerprint"): + verify_review_attestations( + policy, + trust, + reviews, + ReviewAttestationBundle.model_validate(payload), + ) + + +def test_attestation_verifier_rejects_revoked_signing_key() -> None: + policy, _trust, _reviews, _bundle = _signed_attestations() + revoked_trust = build_reviewer_trust( + ReviewerTrustDeclaration.model_validate( + { + **_trust_declaration().model_dump(mode="json"), + "keys": [ + { + **key, + "status": ( + "revoked" + if key["reviewer_id"] == "reviewer-2" + else key["status"] + ), + } + for key in _trust_declaration().model_dump(mode="json")["keys"] + ], + } + ), + policy, + require_active_key_per_reviewer=False, + ) + reviews = _completed_reviews( + reviewer_trust_sha256=revoked_trust.reviewer_trust_sha256 + ) + template = build_review_attestation_template(policy, revoked_trust, reviews) + private_keys = _private_keys() + signed = tuple( + SignedReviewAttestation( + statement=request.statement, + key_id=f"{request.statement.reviewer_id}-key-1", + signature_base64=base64.b64encode( + private_keys[request.statement.reviewer_id].sign( + request.statement.signing_bytes() + ) + ).decode(), + ) + for request in template.requests + ) + bundle = ReviewAttestationBundle( + schema_version=1, + benchmark_id=reviews.benchmark_id, + review_policy_sha256=policy.policy_sha256, + reviewer_trust_sha256=revoked_trust.reviewer_trust_sha256, + reviews_sha256=template.reviews_sha256, + attestations=signed, + ) + + with pytest.raises(ValueError, match="revoked"): + verify_review_attestations(policy, revoked_trust, reviews, bundle) + + +def test_trust_bound_finalization_requires_verified_attestations() -> None: + policy, trust, reviews, attestations = _signed_attestations() + + with pytest.raises(ValueError, match="require policy, trust, and attestations"): + finalize_recorded_results( + _manifest(), + _evidence(), + reviews, + review_policy=policy, + reviewer_trust=trust, + ) + + results = finalize_recorded_results( + _manifest(), + _evidence(), + reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + ) + + assert len(results.observations) == 3 + + +def test_trust_bound_finalize_and_status_cli_require_and_verify_bundle( + tmp_path: Path, +) -> None: + policy, trust, reviews, attestations = _signed_attestations() + paths = { + "manifest": tmp_path / "manifest.json", + "preflight": tmp_path / "preflight.json", + "evidence": tmp_path / "evidence.json", + "reviews": tmp_path / "reviews.json", + "policy": tmp_path / "policy.json", + "trust": tmp_path / "trust.json", + "attestations": tmp_path / "attestations.json", + "results": tmp_path / "results.json", + } + paths["manifest"].write_text(_manifest().model_dump_json(), encoding="utf-8") + paths["preflight"].write_text(_preflight().to_json(), encoding="utf-8") + paths["evidence"].write_text(_evidence().model_dump_json(), encoding="utf-8") + paths["reviews"].write_text(reviews.model_dump_json(), encoding="utf-8") + paths["policy"].write_text( + json.dumps( + { + "schema_version": 1, + "benchmark_id": policy.benchmark_id, + "operator_id": policy.operator_id, + "reviewer_ids": list(policy.reviewer_ids), + "minimum_distinct_reviewers": policy.minimum_distinct_reviewers, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + paths["trust"].write_text(_trust_declaration().model_dump_json(), encoding="utf-8") + paths["attestations"].write_text(attestations.to_json(), encoding="utf-8") + base = [ + "benchmark", + "agent-cli-finalize", + "--manifest", + str(paths["manifest"]), + "--preflight", + str(paths["preflight"]), + "--evidence", + str(paths["evidence"]), + "--reviews", + str(paths["reviews"]), + "--review-policy", + str(paths["policy"]), + "--output", + str(paths["results"]), + ] + + refused = runner.invoke(app, base) + accepted = runner.invoke( + app, + [ + *base, + "--reviewer-trust", + str(paths["trust"]), + "--attestations", + str(paths["attestations"]), + ], + ) + + assert refused.exit_code == 1 + assert "--reviewer-trust is required" in refused.output + assert accepted.exit_code == 0 + before = {path: path.read_bytes() for path in paths.values()} + status = runner.invoke( + app, + [ + "benchmark", + "agent-cli-status", + "--manifest", + str(paths["manifest"]), + "--preflight", + str(paths["preflight"]), + "--evidence", + str(paths["evidence"]), + "--reviews", + str(paths["reviews"]), + "--results", + str(paths["results"]), + "--review-policy", + str(paths["policy"]), + "--reviewer-trust", + str(paths["trust"]), + "--attestations", + str(paths["attestations"]), + "--json", + ], + ) + + assert status.exit_code == 0 + assert json.loads(status.output)["stage"] == "finalized" + assert json.loads(status.output)["attestations_verified"] is True + assert before == {path: path.read_bytes() for path in paths.values()} + + +def test_attestation_template_cli_is_read_only(tmp_path: Path) -> None: + policy, trust, reviews, _bundle = _signed_attestations() + reviews_path = tmp_path / "reviews.json" + policy_path = tmp_path / "policy.json" + trust_path = tmp_path / "trust.json" + output_path = tmp_path / "attestation-template.json" + reviews_path.write_text(reviews.model_dump_json(), encoding="utf-8") + policy_path.write_text( + json.dumps( + { + "schema_version": 1, + "benchmark_id": policy.benchmark_id, + "operator_id": policy.operator_id, + "reviewer_ids": list(policy.reviewer_ids), + "minimum_distinct_reviewers": policy.minimum_distinct_reviewers, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + trust_path.write_text(_trust_declaration().model_dump_json(), encoding="utf-8") + before = {path: path.read_bytes() for path in tmp_path.iterdir()} + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-attestation-template", + "--reviews", + str(reviews_path), + "--review-policy", + str(policy_path), + "--reviewer-trust", + str(trust_path), + "--output", + str(output_path), + "--json", + ], + ) + + after_inputs = {path: path.read_bytes() for path in before} + assert result.exit_code == 0 + assert json.loads(result.output)["attestations_completed"] is False + assert before == after_inputs + assert output_path.exists() + + +def test_committed_reviewer_trust_template_validates() -> None: + root = Path(__file__).parents[3] + payload = json.loads( + (root / "benchmarks/templates/agent_cli_reviewer_trust.example.json").read_text() + ) + policy_payload = json.loads( + (root / "benchmarks/templates/agent_cli_review_policy.example.json").read_text() + ) + policy = build_reviewer_policy(ReviewerPolicyDeclaration.model_validate(policy_payload)) + + trust = build_reviewer_trust(ReviewerTrustDeclaration.model_validate(payload), policy) + + assert trust.benchmark_id == "agent-cli-local-rehearsal" + + +def test_benchmark_authority_is_deterministic_and_self_fingerprinted() -> None: + first = build_benchmark_authority(_authority_declaration()) + second = build_benchmark_authority(_authority_declaration()) + + assert first.to_json() == second.to_json() + assert len(first.public_key_sha256) == 64 + assert len(first.authority_sha256) == 64 + + +def test_reviewer_enrollments_verify_every_trust_key() -> None: + authority, policy, trust, enrollments, *_rest = _anchored_artifacts() + + verify_reviewer_enrollments(authority, policy, trust, enrollments) + + assert [ + certificate.statement.key_id for certificate in enrollments.certificates + ] == ["reviewer-1-key-1", "reviewer-2-key-1"] + assert len(enrollments.reviewer_enrollments_sha256) == 64 + + +def test_reviewer_enrollment_template_cli_is_private_key_free_and_read_only( + tmp_path: Path, +) -> None: + authority, policy, trust, *_rest = _anchored_artifacts() + template = build_reviewer_enrollment_template(authority, policy, trust) + policy_path = tmp_path / "policy.json" + trust_path = tmp_path / "trust.json" + authority_path = tmp_path / "authority.json" + output_path = tmp_path / "enrollment-template.json" + policy_path.write_text( + json.dumps( + { + "schema_version": 1, + "benchmark_id": policy.benchmark_id, + "operator_id": policy.operator_id, + "reviewer_ids": list(policy.reviewer_ids), + "minimum_distinct_reviewers": policy.minimum_distinct_reviewers, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + trust_path.write_text( + _trust_declaration( + reviewer_authority_sha256=authority.authority_sha256 + ).model_dump_json(), + encoding="utf-8", + ) + authority_path.write_text(_authority_declaration().model_dump_json(), encoding="utf-8") + before = {path: path.read_bytes() for path in tmp_path.iterdir()} + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-reviewer-enrollment-template", + "--review-policy", + str(policy_path), + "--reviewer-trust", + str(trust_path), + "--reviewer-authority", + str(authority_path), + "--output", + str(output_path), + "--json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.output) == json.loads(template.to_json()) + assert len(template.requests) == 2 + assert "private" not in template.to_json().lower() + assert before == {path: path.read_bytes() for path in before} + + +@pytest.mark.parametrize("mutation", ["signature", "missing", "mixed_trust"]) +def test_reviewer_enrollments_reject_tampering_and_incomplete_coverage( + mutation: str, +) -> None: + authority, policy, trust, enrollments, *_rest = _anchored_artifacts() + payload = enrollments.model_dump(mode="json") + if mutation == "signature": + payload["certificates"][0]["signature_base64"] = base64.b64encode( + b"x" * 64 + ).decode() + elif mutation == "missing": + payload["certificates"].pop() + else: + payload["reviewer_trust_sha256"] = "f" * 64 + + with pytest.raises( + ValueError, + match="signature|coverage|trust fingerprint|enrollment fingerprint", + ): + verify_reviewer_enrollments( + authority, + policy, + trust, + type(enrollments).model_validate(payload), + ) + + +def test_reviewer_enrollment_rejects_invalid_authority_signature() -> None: + authority, policy, trust, enrollments, *_rest = _anchored_artifacts() + certificates = list(enrollments.certificates) + certificates[0] = certificates[0].model_copy( + update={"signature_base64": base64.b64encode(b"x" * 64).decode()} + ) + + with pytest.raises(ValueError, match="enrollment signature"): + build_reviewer_enrollment_bundle( + authority, + policy, + trust, + tuple(certificates), + ) + + +def test_authority_bound_finalization_requires_enrollment_certificates() -> None: + ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + _results, + _request, + _envelope, + ) = _anchored_artifacts() + + with pytest.raises(ValueError, match="authority and reviewer enrollments"): + finalize_recorded_results( + _manifest(), + _evidence(), + reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + ) + + results = finalize_recorded_results( + _manifest(), + _evidence(), + reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + reviewer_authority=authority, + reviewer_enrollments=enrollments, + ) + + assert len(results.observations) == 3 + + +def test_campaign_envelope_binds_every_artifact_without_authorizing_execution() -> None: + *_, request, _envelope = _anchored_artifacts() + second = _anchored_artifacts()[-2] + + assert request.to_json() == second.to_json() + assert request.statement.paid_execution_authorized is False + assert len(request.statement.results_sha256) == 64 + assert len(request.statement.reviewer_enrollments_sha256) == 64 + assert len(request.statement.attestations_sha256) == 64 + + +def test_signed_campaign_envelope_verifies_and_rejects_tampering() -> None: + ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + results, + _request, + envelope, + ) = _anchored_artifacts() + kwargs = { + "authority": authority, + "manifest": _manifest(), + "preflight": _preflight(), + "evidence": _evidence(), + "reviews": reviews, + "review_policy": policy, + "reviewer_trust": trust, + "reviewer_enrollments": enrollments, + "attestations": attestations, + "results": results, + } + + verify_signed_campaign_envelope(**kwargs, envelope=envelope) + tampered = envelope.model_copy( + update={"signature_base64": base64.b64encode(b"x" * 64).decode()} + ) + with pytest.raises(ValueError, match="campaign envelope signature"): + verify_signed_campaign_envelope(**kwargs, envelope=tampered) + + +def test_authority_bound_campaign_waits_for_signed_envelope_before_finalized() -> None: + ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + results, + _request, + envelope, + ) = _anchored_artifacts() + common = { + "preflight": _preflight(), + "evidence": _evidence(), + "reviews": reviews, + "results": results, + "review_policy": policy, + "reviewer_trust": trust, + "attestations": attestations, + "reviewer_authority": authority, + "reviewer_enrollments": enrollments, + } + + pending = build_campaign_status(_manifest(), **common) + finalized = build_campaign_status( + _manifest(), + **common, + campaign_envelope=envelope, + ) + + assert pending.stage is CampaignStage.CAMPAIGN_ENVELOPE_PENDING + assert pending.next_action == "sign_campaign_envelope" + assert finalized.stage is CampaignStage.FINALIZED + assert finalized.campaign_envelope_verified is True + assert finalized.paid_execution_authorized is False + assert b"authority_root_ledger_sha256" not in envelope.statement.signing_bytes() + + +def test_ledger_bound_campaign_requires_root_then_envelope_inclusion() -> None: + ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + results, + ledger, + envelope, + proof, + _log, + ) = _ledger_bound_artifacts() + review_common = { + "preflight": _preflight(), + "evidence": _evidence(), + "reviews": reviews, + "review_policy": policy, + "reviewer_trust": trust, + "attestations": attestations, + "reviewer_authority": authority, + "reviewer_enrollments": enrollments, + } + + root_pending = build_campaign_status(_manifest(), **review_common) + envelope_pending = build_campaign_status( + _manifest(), + **review_common, + results=results, + authority_root_ledger=ledger, + ) + transparency_pending = build_campaign_status( + _manifest(), + **review_common, + results=results, + authority_root_ledger=ledger, + campaign_envelope=envelope, + ) + finalized = build_campaign_status( + _manifest(), + **review_common, + results=results, + authority_root_ledger=ledger, + campaign_envelope=envelope, + transparency_proof=proof, + ) + + assert root_pending.stage is CampaignStage.AUTHORITY_ROOT_PENDING + assert envelope_pending.stage is CampaignStage.CAMPAIGN_ENVELOPE_PENDING + assert transparency_pending.stage is CampaignStage.TRANSPARENCY_PENDING + assert transparency_pending.campaign_envelope_verified is True + assert finalized.stage is CampaignStage.FINALIZED + assert finalized.authority_root_ledger_verified is True + assert finalized.transparency_inclusion_verified is True + assert finalized.paid_execution_authorized is False + + +def test_witnessed_campaign_requires_consistent_quorum_checkpoint() -> None: + ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + results, + ledger, + envelope, + inclusion_proof, + log, + ) = _ledger_bound_artifacts() + previous_log = build_transparency_log(log.log_id, log.entries[:2]) + previous_request = build_transparency_tree_head_request(previous_log, ledger) + previous_head = SignedTransparencyTreeHead( + statement=previous_request.statement, + signature_base64=base64.b64encode( + _authority_private_key().sign(previous_request.statement.signing_bytes()) + ).decode(), + ) + consistency_proof = build_transparency_consistency_proof( + log, + previous_tree_head=previous_head, + current_tree_head=inclusion_proof.tree_head, + authority_root_ledger=ledger, + ) + witness_private_keys = { + f"witness-{index}": Ed25519PrivateKey.from_private_bytes(bytes([index + 4]) * 32) + for index in range(1, 4) + } + witness_trust = build_transparency_witness_trust( + TransparencyWitnessTrustDeclaration( + schema_version=1, + log_id=log.log_id, + minimum_distinct_witnesses=2, + keys=tuple( + TransparencyWitnessKeyDeclaration( + witness_id=witness_id, + key_id=f"{witness_id}-key-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + ) + for witness_id, private_key in witness_private_keys.items() + ), + ) + ) + template = build_witness_checkpoint_template( + witness_trust, + consistency_proof, + ledger, + ) + signatures = tuple( + TransparencyWitnessSignature( + witness_id=request.witness_id, + key_id=f"{request.witness_id}-key-1", + signature_base64=base64.b64encode( + witness_private_keys[request.witness_id].sign( + request.statement.signing_bytes() + ) + ).decode(), + ) + for request in template.requests[:2] + ) + checkpoint = build_witness_checkpoint_bundle( + witness_trust, + consistency_proof, + ledger, + signatures, + ) + common = { + "preflight": _preflight(), + "evidence": _evidence(), + "reviews": reviews, + "results": results, + "review_policy": policy, + "reviewer_trust": trust, + "attestations": attestations, + "reviewer_authority": authority, + "reviewer_enrollments": enrollments, + "authority_root_ledger": ledger, + "campaign_envelope": envelope, + "transparency_proof": inclusion_proof, + "transparency_witness_trust": witness_trust, + } + + pending = build_campaign_status(_manifest(), **common) + finalized = build_campaign_status( + _manifest(), + **common, + transparency_consistency_proof=consistency_proof, + witness_checkpoint=checkpoint, + ) + + assert pending.stage is CampaignStage.WITNESS_PENDING + assert pending.next_action == "collect_witness_checkpoint" + assert finalized.stage is CampaignStage.FINALIZED + assert finalized.transparency_consistency_verified is True + assert finalized.witness_checkpoint_verified is True + assert ( + finalized.transparency_consistency_proof_sha256 + == consistency_proof.consistency_proof_sha256 + ) + assert ( + finalized.witness_checkpoint_sha256 + == checkpoint.witness_checkpoint_sha256 + ) + assert finalized.paid_execution_authorized is False + + +def test_ledger_bound_results_reject_a_different_signed_root_ledger() -> None: + ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + _results, + ledger, + _envelope, + _proof, + _log, + ) = _ledger_bound_artifacts() + changed_statement = ledger.statement.model_copy( + update={"ledger_sha256": "f" * 64} + ) + changed = ledger.model_copy(update={"statement": changed_statement}) + + with pytest.raises(ValueError, match="ledger fingerprint"): + finalize_recorded_results( + _manifest(), + _evidence(), + reviews, + review_policy=policy, + reviewer_trust=trust, + attestations=attestations, + reviewer_authority=authority, + reviewer_enrollments=enrollments, + authority_root_ledger=changed, + ) + + +def test_campaign_envelope_template_cli_preserves_all_inputs(tmp_path: Path) -> None: + ( + authority, + policy, + trust, + enrollments, + reviews, + attestations, + results, + request, + envelope, + ) = _anchored_artifacts() + payloads = { + "manifest": _manifest().model_dump_json(), + "preflight": _preflight().to_json(), + "evidence": _evidence().model_dump_json(), + "reviews": reviews.model_dump_json(), + "policy": json.dumps( + { + "schema_version": 1, + "benchmark_id": policy.benchmark_id, + "operator_id": policy.operator_id, + "reviewer_ids": list(policy.reviewer_ids), + "minimum_distinct_reviewers": policy.minimum_distinct_reviewers, + }, + sort_keys=True, + ), + "trust": _trust_declaration( + reviewer_authority_sha256=authority.authority_sha256 + ).model_dump_json(), + "authority": _authority_declaration().model_dump_json(), + "enrollments": enrollments.to_json(), + "attestations": attestations.to_json(), + "results": results.model_dump_json(), + } + paths = {} + for name, payload in payloads.items(): + paths[name] = tmp_path / f"{name}.json" + paths[name].write_text(payload, encoding="utf-8") + finalized = tmp_path / "finalized.json" + before_finalize = {path: path.read_bytes() for path in paths.values()} + finalize_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-finalize", + "--manifest", + str(paths["manifest"]), + "--preflight", + str(paths["preflight"]), + "--evidence", + str(paths["evidence"]), + "--reviews", + str(paths["reviews"]), + "--review-policy", + str(paths["policy"]), + "--reviewer-trust", + str(paths["trust"]), + "--reviewer-authority", + str(paths["authority"]), + "--reviewer-enrollments", + str(paths["enrollments"]), + "--attestations", + str(paths["attestations"]), + "--output", + str(finalized), + ], + ) + + assert finalize_result.exit_code == 0 + assert json.loads(finalized.read_text()) == results.model_dump(mode="json") + assert before_finalize == {path: path.read_bytes() for path in paths.values()} + output = tmp_path / "campaign-envelope-template.json" + before = {path: path.read_bytes() for path in paths.values()} + args = ["benchmark", "agent-cli-campaign-envelope-template"] + for name in payloads: + option = "--review-policy" if name == "policy" else f"--{name}" + option = "--reviewer-trust" if name == "trust" else option + option = "--reviewer-authority" if name == "authority" else option + option = "--reviewer-enrollments" if name == "enrollments" else option + args.extend([option, str(paths[name])]) + args.extend(["--output", str(output), "--json"]) + + result = runner.invoke(app, args) + + assert result.exit_code == 0 + assert json.loads(result.output) == json.loads(request.to_json()) + assert before == {path: path.read_bytes() for path in paths.values()} + assert output.exists() + envelope_path = tmp_path / "campaign-envelope.json" + envelope_path.write_text(envelope.to_json(), encoding="utf-8") + status_before = { + path: path.read_bytes() for path in [*paths.values(), envelope_path] + } + status_args = ["benchmark", "agent-cli-status"] + for name in payloads: + option = "--review-policy" if name == "policy" else f"--{name}" + option = "--reviewer-trust" if name == "trust" else option + option = "--reviewer-authority" if name == "authority" else option + option = "--reviewer-enrollments" if name == "enrollments" else option + status_args.extend([option, str(paths[name])]) + status_args.extend( + ["--campaign-envelope", str(envelope_path), "--json"] + ) + + status = runner.invoke(app, status_args) + + assert status.exit_code == 0 + assert json.loads(status.output)["stage"] == "finalized" + assert json.loads(status.output)["campaign_envelope_verified"] is True + assert status_before == { + path: path.read_bytes() for path in [*paths.values(), envelope_path] + } + + +def test_committed_authority_and_anchored_trust_templates_validate() -> None: + root = Path(__file__).parents[3] + authority_declaration = BenchmarkAuthorityDeclaration.model_validate_json( + (root / "benchmarks/templates/agent_cli_reviewer_authority.example.json").read_text() + ) + trust_declaration = ReviewerTrustDeclaration.model_validate_json( + ( + root + / "benchmarks/templates/agent_cli_anchored_reviewer_trust.example.json" + ).read_text() + ) + policy = build_reviewer_policy( + ReviewerPolicyDeclaration.model_validate_json( + (root / "benchmarks/templates/agent_cli_review_policy.example.json").read_text() + ) + ) + authority = build_benchmark_authority(authority_declaration) + + trust = build_reviewer_trust(trust_declaration, policy) + + assert trust.reviewer_authority_sha256 == authority.authority_sha256 diff --git a/tests/unit/benchmarks/test_agent_cli_recorder.py b/tests/unit/benchmarks/test_agent_cli_recorder.py new file mode 100644 index 0000000..2dee229 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_recorder.py @@ -0,0 +1,389 @@ +"""Tests for the opt-in isolated agent CLI trial recorder.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from benchmarks.agent_cli_comparison import AgentCliManifest +from benchmarks.agent_cli_receipts import ProviderReceipt +from benchmarks.agent_cli_recorder import ( + AgentCliRecorderConfig, + AgentCliTrialRecorder, + CommandCapture, + GitWorktreeManager, + LocalCommandRunner, + build_recording_plan, + validate_execution_consent, +) +from interface.cli.commands.benchmark import _write_new_evidence +from interface.cli.main import app + +runner = CliRunner() + + +def _manifest() -> dict[str, object]: + return { + "schema_version": 1, + "benchmark_id": "record-001", + "task": { + "id": "task-001", + "goal": "Implement one isolated change", + "workspace_revision": "abc123", + "checks": ["unit"], + "handoff_assertions": ["decision"], + }, + "arms": ["codex_cli", "claude_code", "morphic_control"], + "repetitions": 1, + } + + +def _config() -> dict[str, object]: + return { + "schema_version": 1, + "benchmark_id": "record-001", + "arm_commands": { + "codex_cli": ["codex", "{goal}"], + "claude_code": ["claude", "{goal}"], + "morphic_control": ["morphic", "code", "{goal}"], + }, + "check_commands": {"unit": ["verify", "unit"]}, + "handoff_commands": {"decision": ["verify", "decision"]}, + "estimated_cost_usd_per_trial": { + "codex_cli": 0.1, + "claude_code": 0.2, + "morphic_control": 0.3, + }, + "timeout_seconds": 60.0, + } + + +class FakeWorktrees: + def __init__(self) -> None: + self.created: list[tuple[Path, str, Path]] = [] + self.released: list[Path] = [] + + async def create(self, *, source_root: Path, revision: str, destination: Path) -> None: + self.created.append((source_root, revision, destination)) + destination.mkdir(parents=True) + + async def release(self, *, source_root: Path, destination: Path) -> None: + self.released.append(destination) + + +class FakeCommands: + def __init__(self, *, fail_with: Exception | None = None) -> None: + self.calls: list[tuple[tuple[str, ...], Path, float]] = [] + self.fail_with = fail_with + + async def run( + self, + *, + argv: tuple[str, ...], + cwd: Path, + timeout_seconds: float, + ) -> CommandCapture: + self.calls.append((argv, cwd, timeout_seconds)) + if self.fail_with is not None: + raise self.fail_with + return CommandCapture( + exit_code=0, + stdout="secret output", + stderr="secret warning", + elapsed_seconds=1.25, + timed_out=False, + ) + + +class FakeReceipts: + def parse( + self, + *, + arm: object, + stdout: str, + model_hint: str | None = None, + ) -> ProviderReceipt: + return ProviderReceipt( + provider=getattr(arm, "value", arm), + success=True, + model=model_hint or "test-model", + usage={"input_tokens": 10, "output_tokens": 2}, + cost_usd=(0.00006 if getattr(arm, "value", arm) == "codex_cli" else 0.01), + cost_source={ + "codex_cli": "calculated_from_usage", + "claude_code": "provider_reported", + "morphic_control": "morphic_reported", + }[getattr(arm, "value", arm)], + ) + + +def test_build_recording_plan_is_deterministic_and_reserves_cost() -> None: + manifest = AgentCliManifest.model_validate(_manifest()) + config = AgentCliRecorderConfig.model_validate(_config()) + + first = build_recording_plan(manifest, config).to_json() + second = build_recording_plan(manifest, config).to_json() + payload = json.loads(first) + + assert first == second + assert payload["trial_count"] == 3 + assert payload["estimated_max_cost_usd"] == 0.6 + assert payload["execution_required"] is False + assert "Implement one isolated change" not in first + assert "timestamp" not in payload + + +@pytest.mark.parametrize( + ("acknowledged_paid", "cost_cap_usd", "match"), + [ + (False, 1.0, "acknowledgement"), + (True, None, "cost cap"), + (True, 0.59, "below estimated"), + ], +) +def test_execution_consent_fails_closed( + acknowledged_paid: bool, + cost_cap_usd: float | None, + match: str, +) -> None: + plan = build_recording_plan( + AgentCliManifest.model_validate(_manifest()), + AgentCliRecorderConfig.model_validate(_config()), + ) + + with pytest.raises(ValueError, match=match): + validate_execution_consent( + plan, + acknowledged_paid=acknowledged_paid, + cost_cap_usd=cost_cap_usd, + ) + + +def test_config_requires_exact_declared_arms_checks_and_handoffs() -> None: + manifest = AgentCliManifest.model_validate(_manifest()) + raw = _config() + del raw["arm_commands"]["claude_code"] # type: ignore[index] + + with pytest.raises(ValueError, match="arm_commands"): + build_recording_plan(manifest, AgentCliRecorderConfig.model_validate(raw)) + + raw = _config() + raw["check_commands"] = {"other": ["verify"]} + with pytest.raises(ValueError, match="check_commands"): + build_recording_plan(manifest, AgentCliRecorderConfig.model_validate(raw)) + + +@pytest.mark.asyncio +async def test_recorder_isolates_trials_and_hashes_raw_output(tmp_path: Path) -> None: + worktrees = FakeWorktrees() + commands = FakeCommands() + recorder = AgentCliTrialRecorder(worktree_manager=worktrees, command_runner=commands) + + evidence = await recorder.record( + manifest=AgentCliManifest.model_validate(_manifest()), + config=AgentCliRecorderConfig.model_validate(_config()), + source_root=tmp_path / "source", + worktree_root=tmp_path / "worktrees", + acknowledged_paid=True, + cost_cap_usd=0.6, + ) + payload = evidence.to_dict() + + assert len(worktrees.created) == 3 + assert len({item[2] for item in worktrees.created}) == 3 + assert worktrees.released == [item[2] for item in worktrees.created] + assert len(payload["trials"]) == 3 + assert payload["authorized_cost_cap_usd"] == 0.6 + assert payload["trials"][0]["passed_checks"] == ["unit"] + assert payload["trials"][0]["passed_handoff_assertions"] == ["decision"] + assert "secret output" not in evidence.to_json() + assert "secret warning" not in evidence.to_json() + assert payload["trials"][0]["agent"]["stdout_bytes"] == 13 + assert len(payload["trials"][0]["agent"]["stdout_sha256"]) == 64 + assert payload["cost_collection"] == "pending_adjudication" + + +@pytest.mark.asyncio +async def test_recorder_normalizes_all_receipts_before_discarding_output(tmp_path: Path) -> None: + recorder = AgentCliTrialRecorder( + worktree_manager=FakeWorktrees(), + command_runner=FakeCommands(), + receipt_parser=FakeReceipts(), + ) + + evidence = await recorder.record( + manifest=AgentCliManifest.model_validate(_manifest()), + config=AgentCliRecorderConfig.model_validate(_config()), + source_root=tmp_path / "source", + worktree_root=tmp_path / "worktrees", + acknowledged_paid=True, + cost_cap_usd=0.6, + ) + payload = evidence.to_dict() + + assert payload["cost_collection"] == "normalized_receipts" + assert payload["trials"][0]["receipt"]["cost_usd"] == 0.00006 + assert "secret output" not in evidence.to_json() + + +@pytest.mark.asyncio +async def test_recorder_releases_worktree_when_command_raises(tmp_path: Path) -> None: + worktrees = FakeWorktrees() + recorder = AgentCliTrialRecorder( + worktree_manager=worktrees, + command_runner=FakeCommands(fail_with=RuntimeError("runner failed")), + ) + + with pytest.raises(RuntimeError, match="runner failed"): + await recorder.record( + manifest=AgentCliManifest.model_validate(_manifest()), + config=AgentCliRecorderConfig.model_validate(_config()), + source_root=tmp_path / "source", + worktree_root=tmp_path / "worktrees", + acknowledged_paid=True, + cost_cap_usd=0.6, + ) + + assert len(worktrees.created) == 1 + assert worktrees.released == [worktrees.created[0][2]] + + +@pytest.mark.asyncio +async def test_recorder_refuses_worktrees_inside_source(tmp_path: Path) -> None: + recorder = AgentCliTrialRecorder( + worktree_manager=FakeWorktrees(), + command_runner=FakeCommands(), + ) + source_root = tmp_path / "source" + + with pytest.raises(ValueError, match="outside source_root"): + await recorder.record( + manifest=AgentCliManifest.model_validate(_manifest()), + config=AgentCliRecorderConfig.model_validate(_config()), + source_root=source_root, + worktree_root=source_root / ".morphic" / "worktrees", + acknowledged_paid=True, + cost_cap_usd=0.6, + ) + + assert not (source_root / ".morphic").exists() + + +@pytest.mark.asyncio +async def test_local_runner_passes_arguments_without_a_shell(tmp_path: Path) -> None: + marker = tmp_path / "must-not-exist" + literal = f"$(touch {marker})" + + capture = await LocalCommandRunner().run( + argv=(sys.executable, "-c", "import sys; print(sys.argv[1])", literal), + cwd=tmp_path, + timeout_seconds=5.0, + ) + + assert capture.exit_code == 0 + assert capture.stdout.strip() == literal + assert not marker.exists() + + +@pytest.mark.asyncio +async def test_git_worktree_manager_creates_pinned_detached_workspace(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + command_runner = LocalCommandRunner() + + async def git(*args: str, cwd: Path = source) -> CommandCapture: + capture = await command_runner.run( + argv=("git", *args), + cwd=cwd, + timeout_seconds=10.0, + ) + assert capture.exit_code == 0, capture.stderr + return capture + + await git("init") + await git("config", "user.email", "benchmark@example.invalid") + await git("config", "user.name", "Benchmark Test") + (source / "evidence.txt").write_text("pinned", encoding="utf-8") + await git("add", "evidence.txt") + await git("commit", "-m", "Create pinned revision") + revision = (await git("rev-parse", "HEAD")).stdout.strip() + destination = tmp_path / "worktrees" / "trial" + destination.parent.mkdir() + manager = GitWorktreeManager() + + await manager.create(source_root=source, revision=revision, destination=destination) + + assert (destination / "evidence.txt").read_text(encoding="utf-8") == "pinned" + assert (await git("rev-parse", "HEAD", cwd=destination)).stdout.strip() == revision + + await manager.release(source_root=source, destination=destination) + assert not destination.exists() + + +def test_evidence_publish_is_exclusive_and_leaves_no_temporary_file(tmp_path: Path) -> None: + evidence = tmp_path / "evidence.json" + _write_new_evidence(evidence, "first") + + with pytest.raises(FileExistsError): + _write_new_evidence(evidence, "second") + + assert evidence.read_text(encoding="utf-8") == "first" + assert list(tmp_path.glob(".evidence.json.*.tmp")) == [] + + +def test_cli_defaults_to_plan_without_creating_worktrees(tmp_path: Path) -> None: + manifest_path = tmp_path / "manifest.json" + config_path = tmp_path / "recorder.json" + manifest_path.write_text(json.dumps(_manifest()), encoding="utf-8") + config_path.write_text(json.dumps(_config()), encoding="utf-8") + worktree_root = tmp_path / "worktrees" + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-record", + "--manifest", + str(manifest_path), + "--config", + str(config_path), + "--json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.output)["execution_required"] is False + assert not worktree_root.exists() + + +def test_cli_rejects_execute_without_paid_acknowledgement(tmp_path: Path) -> None: + manifest_path = tmp_path / "manifest.json" + config_path = tmp_path / "recorder.json" + manifest_path.write_text(json.dumps(_manifest()), encoding="utf-8") + config_path.write_text(json.dumps(_config()), encoding="utf-8") + worktree_root = tmp_path / "worktrees" + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-record", + "--manifest", + str(manifest_path), + "--config", + str(config_path), + "--worktree-root", + str(worktree_root), + "--execute", + "--cost-cap-usd", + "1.0", + ], + ) + + assert result.exit_code == 2 + assert "acknowledgement" in result.output + assert not worktree_root.exists() diff --git a/tests/unit/benchmarks/test_agent_cli_rehearsal.py b/tests/unit/benchmarks/test_agent_cli_rehearsal.py new file mode 100644 index 0000000..9292146 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_rehearsal.py @@ -0,0 +1,125 @@ +"""Tests for zero-cost agent CLI benchmark rehearsal.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from benchmarks.agent_cli_comparison import AgentCliArm, AgentCliManifest +from benchmarks.agent_cli_recorder import AgentCliRecorderConfig, LocalCommandRunner +from benchmarks.agent_cli_rehearsal import ( + build_local_rehearsal_contract, + publish_local_rehearsal, + run_local_rehearsal, +) +from interface.cli.main import app + +runner = CliRunner() + + +class _DirectoryWorktrees: + async def create(self, *, source_root: Path, revision: str, destination: Path) -> None: + del source_root, revision + destination.mkdir(parents=True) + + async def release(self, *, source_root: Path, destination: Path) -> None: + del source_root + destination.rmdir() + + +def test_committed_dry_run_templates_validate() -> None: + root = Path(__file__).parents[3] + manifest_payload = json.loads( + (root / "benchmarks/templates/agent_cli_manifest.example.json").read_text() + ) + manifest_payload["task"]["workspace_revision"] = "a" * 40 + config_payload = json.loads( + (root / "benchmarks/templates/agent_cli_recorder.example.json").read_text() + ) + + manifest = AgentCliManifest.model_validate(manifest_payload) + example_config = AgentCliRecorderConfig.model_validate(config_payload) + _, config = build_local_rehearsal_contract( + workspace_revision=manifest.task.workspace_revision, + python_executable=sys.executable, + benchmark_id=manifest.benchmark_id, + task_id=manifest.task.id, + goal=manifest.task.goal, + ) + + assert set(config_payload["arm_commands"]) == { + "codex_cli", + "claude_code", + "morphic_control", + } + assert set(example_config.estimated_cost_usd_per_trial.values()) == {0.0} + assert config.benchmark_id == manifest.benchmark_id + + +def test_local_rehearsal_contract_is_internal_and_zero_cost() -> None: + manifest, config = build_local_rehearsal_contract( + workspace_revision="revision-43", + python_executable=sys.executable, + ) + + assert manifest.repetitions == 1 + assert set(config.arm_commands) == set(AgentCliArm) + assert set(config.estimated_cost_usd_per_trial.values()) == {0.0} + assert all(command[:2] == (sys.executable, "-c") for command in config.arm_commands.values()) + assert all("codex" not in command[0] for command in config.arm_commands.values()) + assert all("claude" not in command[0] for command in config.arm_commands.values()) + + +@pytest.mark.asyncio +async def test_local_rehearsal_runs_complete_pipeline_without_agent_processes( + tmp_path: Path, +) -> None: + artifacts = await run_local_rehearsal( + source_root=tmp_path / "source", + worktree_root=tmp_path / "worktrees", + workspace_revision="revision-43", + python_executable=sys.executable, + worktree_manager=_DirectoryWorktrees(), + command_runner=LocalCommandRunner(), + ) + + assert artifacts.evidence.cost_collection == "normalized_receipts" + assert len(artifacts.evidence.trials) == 3 + assert {trial.receipt.cost_usd for trial in artifacts.evidence.trials if trial.receipt} == { + 0.0 + } + assert len(artifacts.results.observations) == 3 + assert all(observation.completed for observation in artifacts.results.observations) + assert not any(observation.accepted_patch for observation in artifacts.results.observations) + assert not (tmp_path / "worktrees").exists() or not any( + (tmp_path / "worktrees").iterdir() + ) + + +def test_local_rehearsal_publication_is_deterministic_and_exclusive(tmp_path: Path) -> None: + output = tmp_path / "existing" + output.mkdir() + marker = output / "keep.txt" + marker.write_text("preserve", encoding="utf-8") + + with pytest.raises(FileExistsError): + publish_local_rehearsal(output, object()) # type: ignore[arg-type] + + assert marker.read_text(encoding="utf-8") == "preserve" + + +def test_agent_cli_rehearse_cli_refuses_existing_output() -> None: + with runner.isolated_filesystem(): + output = Path("artifacts") + output.mkdir() + result = runner.invoke( + app, + ["benchmark", "agent-cli-rehearse", "--output-dir", str(output)], + ) + + assert result.exit_code == 1 + assert "already exists" in result.output diff --git a/tests/unit/benchmarks/test_agent_cli_transparency.py b/tests/unit/benchmarks/test_agent_cli_transparency.py new file mode 100644 index 0000000..a2fccde --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_transparency.py @@ -0,0 +1,347 @@ +"""Authority-root rotation and append-only campaign transparency tests.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from typer.testing import CliRunner + +from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + build_benchmark_authority, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_transparency import ( + AuthorityRootGeneration, + AuthorityRotationCertificate, + SignedAuthorityRootLedger, + SignedTransparencyTreeHead, + TransparencyInclusionProof, + TransparencyLog, + TransparencyLogEntry, + TransparencyTreeHeadSigningRequest, + build_authority_root_ledger_request, + build_authority_rotation_request, + build_transparency_inclusion_proof, + build_transparency_log, + build_transparency_tree_head_request, + extend_transparency_log, + verify_authority_root_ledger, + verify_complete_log_extension, + verify_transparency_inclusion_proof, +) +from interface.cli.main import app + +runner = CliRunner() + + +def _private(seed: int) -> Ed25519PrivateKey: + return Ed25519PrivateKey.from_private_bytes(bytes([seed]) * 32) + + +def _authority(seed: int, authority_id: str): + private_key = _private(seed) + public_key = private_key.public_key().public_bytes_raw() + authority = build_benchmark_authority( + BenchmarkAuthorityDeclaration( + schema_version=SCHEMA_VERSION, + authority_id=authority_id, + public_key_base64=base64.b64encode(public_key).decode(), + ) + ) + return private_key, authority + + +def _signed_ledger(*, revoke_genesis: bool = False): + genesis_private, genesis = _authority(11, "benchmark-root-2026") + active_private, active = _authority(12, "benchmark-root-2027") + rotation_request = build_authority_rotation_request( + generation=2, + predecessor=genesis, + successor=active, + ) + rotation = AuthorityRotationCertificate( + statement=rotation_request.statement, + signature_base64=base64.b64encode( + genesis_private.sign(rotation_request.statement.signing_bytes()) + ).decode(), + ) + generations = ( + AuthorityRootGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + authority=genesis, + ), + AuthorityRootGeneration( + schema_version=SCHEMA_VERSION, + generation=2, + authority=active, + rotation=rotation, + ), + ) + request = build_authority_root_ledger_request( + generations, + revoked_authority_sha256=(genesis.authority_sha256,) if revoke_genesis else (), + ) + ledger = SignedAuthorityRootLedger( + statement=request.statement, + signature_base64=base64.b64encode( + active_private.sign(request.statement.signing_bytes()) + ).decode(), + ) + return active_private, genesis, active, ledger + + +def _signed_head(log, private_key, ledger): + request = build_transparency_tree_head_request(log, ledger) + return SignedTransparencyTreeHead( + statement=request.statement, + signature_base64=base64.b64encode( + private_key.sign(request.statement.signing_bytes()) + ).decode(), + ) + + +def test_authority_root_ledger_verifies_rotation_revocation_and_active_signature() -> None: + _, genesis, active, ledger = _signed_ledger(revoke_genesis=True) + + verified = verify_authority_root_ledger(ledger) + + assert verified == active + assert ledger.statement.active_generation == 2 + assert ledger.statement.revoked_authority_sha256 == (genesis.authority_sha256,) + + +def test_authority_root_ledger_rejects_invalid_rotation_signature() -> None: + _, _, _, ledger = _signed_ledger() + generation = ledger.statement.generations[1] + tampered_rotation = generation.rotation.model_copy( + update={"signature_base64": base64.b64encode(bytes(64)).decode()} + ) + tampered_generation = generation.model_copy(update={"rotation": tampered_rotation}) + with pytest.raises(ValueError, match="rotation signature is invalid"): + build_authority_root_ledger_request( + (ledger.statement.generations[0], tampered_generation) + ) + + +def test_authority_root_ledger_rejects_revoked_active_root() -> None: + _, _, active, ledger = _signed_ledger() + + with pytest.raises(ValueError, match="active authority is revoked"): + build_authority_root_ledger_request( + ledger.statement.generations, + revoked_authority_sha256=(active.authority_sha256,), + ) + + +def test_transparency_log_uses_rfc6962_domain_separated_hashes() -> None: + first = TransparencyLogEntry( + sequence=0, + kind="authority_root_ledger", + artifact_sha256="11" * 32, + ) + second = TransparencyLogEntry( + sequence=1, + kind="campaign_envelope", + artifact_sha256="22" * 32, + ) + log = build_transparency_log("benchmark-production", (first, second)) + left = hashlib.sha256(b"\x00" + first.leaf_bytes()).digest() + right = hashlib.sha256(b"\x00" + second.leaf_bytes()).digest() + + assert log.root_sha256 == hashlib.sha256(b"\x01" + left + right).hexdigest() + assert log.tree_size == 2 + + +def test_transparency_inclusion_proof_verifies_and_rejects_tampering() -> None: + active_private, _, _, ledger = _signed_ledger(revoke_genesis=True) + entries = tuple( + TransparencyLogEntry( + sequence=index, + kind=kind, + artifact_sha256=f"{index + 1:02x}" * 32, + ) + for index, kind in enumerate( + ("authority_root_ledger", "reviewer_enrollments", "campaign_envelope") + ) + ) + log = build_transparency_log("benchmark-production", entries) + head = _signed_head(log, active_private, ledger) + proof = build_transparency_inclusion_proof(log, leaf_index=2, tree_head=head) + + verify_transparency_inclusion_proof( + proof, + ledger, + expected_kind="campaign_envelope", + expected_artifact_sha256=entries[2].artifact_sha256, + ) + tampered = proof.model_copy( + update={"audit_path_sha256": ("00" * 32,) + proof.audit_path_sha256[1:]} + ) + with pytest.raises(ValueError, match="inclusion proof root does not match"): + verify_transparency_inclusion_proof( + tampered, + ledger, + expected_kind="campaign_envelope", + expected_artifact_sha256=entries[2].artifact_sha256, + ) + + +def test_complete_log_extension_requires_exact_prefix() -> None: + first = TransparencyLogEntry( + sequence=0, + kind="authority_root_ledger", + artifact_sha256="11" * 32, + ) + second = TransparencyLogEntry( + sequence=1, + kind="campaign_envelope", + artifact_sha256="22" * 32, + ) + original = build_transparency_log("benchmark-production", (first,)) + extended = extend_transparency_log(original, (second,)) + + verify_complete_log_extension(original, extended) + replaced = build_transparency_log( + "benchmark-production", + ( + first.model_copy(update={"artifact_sha256": "33" * 32}), + second, + ), + ) + with pytest.raises(ValueError, match="not an append-only extension"): + verify_complete_log_extension(original, replaced) + + +def test_transparency_cli_builds_private_key_free_artifacts_read_only( + tmp_path: Path, +) -> None: + active_private, genesis, _, ledger = _signed_ledger(revoke_genesis=True) + generations_path = tmp_path / "generations.json" + generations_path.write_text( + json.dumps( + { + "generations": [ + item.model_dump(mode="json") + for item in ledger.statement.generations + ], + "revoked_authority_sha256": [genesis.authority_sha256], + }, + sort_keys=True, + ), + encoding="utf-8", + ) + ledger_request_path = tmp_path / "ledger-request.json" + ledger_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-authority-root-ledger-template", + "--generations", + str(generations_path), + "--output", + str(ledger_request_path), + "--json", + ], + ) + assert ledger_result.exit_code == 0 + assert "private" not in ledger_result.output.lower() + + entries_path = tmp_path / "entries.json" + entries_path.write_text( + json.dumps( + [ + { + "sequence": 0, + "kind": "campaign_envelope", + "artifact_sha256": "44" * 32, + } + ], + sort_keys=True, + ), + encoding="utf-8", + ) + log_path = tmp_path / "log.json" + log_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-transparency-log", + "--log-id", + "benchmark-production", + "--entries", + str(entries_path), + "--output", + str(log_path), + "--json", + ], + ) + assert log_result.exit_code == 0 + log = TransparencyLog.model_validate_json(log_path.read_text(encoding="utf-8")) + assert log.tree_size == 1 + + ledger_path = tmp_path / "ledger.json" + ledger_path.write_text(ledger.to_json(), encoding="utf-8") + head_request_path = tmp_path / "head-request.json" + head_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-transparency-tree-head-template", + "--log", + str(log_path), + "--authority-root-ledger", + str(ledger_path), + "--output", + str(head_request_path), + "--json", + ], + ) + assert head_result.exit_code == 0 + request = TransparencyTreeHeadSigningRequest.model_validate_json( + head_request_path.read_text(encoding="utf-8") + ) + head = SignedTransparencyTreeHead( + statement=request.statement, + signature_base64=base64.b64encode( + active_private.sign(request.statement.signing_bytes()) + ).decode(), + ) + head_path = tmp_path / "head.json" + head_path.write_text(head.model_dump_json(), encoding="utf-8") + proof_path = tmp_path / "proof.json" + proof_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-transparency-proof", + "--log", + str(log_path), + "--tree-head", + str(head_path), + "--leaf-index", + "0", + "--output", + str(proof_path), + "--json", + ], + ) + + assert proof_result.exit_code == 0 + proof = TransparencyInclusionProof.model_validate_json( + proof_path.read_text(encoding="utf-8") + ) + verify_transparency_inclusion_proof( + proof, + ledger, + expected_kind="campaign_envelope", + expected_artifact_sha256="44" * 32, + ) + assert generations_path.read_text(encoding="utf-8").startswith("{") + assert entries_path.read_text(encoding="utf-8").startswith("[") diff --git a/tests/unit/benchmarks/test_agent_cli_witness.py b/tests/unit/benchmarks/test_agent_cli_witness.py new file mode 100644 index 0000000..3b34ad4 --- /dev/null +++ b/tests/unit/benchmarks/test_agent_cli_witness.py @@ -0,0 +1,433 @@ +"""Compact Merkle consistency and witnessed checkpoint tests.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import math +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from typer.testing import CliRunner + +from benchmarks.agent_cli_authority import ( + BenchmarkAuthorityDeclaration, + build_benchmark_authority, +) +from benchmarks.agent_cli_comparison import SCHEMA_VERSION +from benchmarks.agent_cli_transparency import ( + AuthorityRootGeneration, + SignedAuthorityRootLedger, + SignedTransparencyTreeHead, + TransparencyConsistencyProof, + TransparencyLogEntry, + build_authority_root_ledger_request, + build_transparency_consistency_proof, + build_transparency_log, + build_transparency_tree_head_request, + verify_transparency_consistency_proof, +) +from benchmarks.agent_cli_witness import ( + SignedWitnessCheckpoint, + TransparencyWitnessKeyDeclaration, + TransparencyWitnessSignature, + TransparencyWitnessTrustDeclaration, + WitnessCheckpointTemplate, + build_transparency_witness_trust, + build_witness_checkpoint_bundle, + build_witness_checkpoint_template, + detect_witness_checkpoint_conflict, + verify_witness_checkpoint_bundle, +) +from interface.cli.main import app + +runner = CliRunner() + + +def _private(seed: int) -> Ed25519PrivateKey: + return Ed25519PrivateKey.from_private_bytes(bytes([seed]) * 32) + + +def _sha256(payload: object) -> str: + encoded = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _authority_ledger(): + private_key = _private(20) + authority = build_benchmark_authority( + BenchmarkAuthorityDeclaration( + schema_version=SCHEMA_VERSION, + authority_id="witness-test-root", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + ) + ) + request = build_authority_root_ledger_request( + ( + AuthorityRootGeneration( + schema_version=SCHEMA_VERSION, + generation=1, + authority=authority, + ), + ) + ) + ledger = SignedAuthorityRootLedger( + statement=request.statement, + signature_base64=base64.b64encode( + private_key.sign(request.statement.signing_bytes()) + ).decode(), + ) + return private_key, ledger + + +def _entries(count: int, *, offset: int = 0) -> tuple[TransparencyLogEntry, ...]: + return tuple( + TransparencyLogEntry( + sequence=index, + kind="campaign_envelope", + artifact_sha256=f"{index + offset + 1:064x}", + ) + for index in range(count) + ) + + +def _signed_head(log, private_key, ledger): + request = build_transparency_tree_head_request(log, ledger) + return SignedTransparencyTreeHead( + statement=request.statement, + signature_base64=base64.b64encode( + private_key.sign(request.statement.signing_bytes()) + ).decode(), + ) + + +def _proof(previous_size: int, current_size: int): + authority_private, ledger = _authority_ledger() + current_log = build_transparency_log("witness-log", _entries(current_size)) + previous_log = build_transparency_log( + "witness-log", + current_log.entries[:previous_size], + ) + previous_head = _signed_head(previous_log, authority_private, ledger) + current_head = _signed_head(current_log, authority_private, ledger) + proof = build_transparency_consistency_proof( + current_log, + previous_tree_head=previous_head, + current_tree_head=current_head, + authority_root_ledger=ledger, + ) + return authority_private, ledger, current_log, proof + + +def _witness_trust(): + keys = {f"witness-{index}": _private(30 + index) for index in range(1, 4)} + declaration = TransparencyWitnessTrustDeclaration( + schema_version=SCHEMA_VERSION, + log_id="witness-log", + minimum_distinct_witnesses=2, + keys=tuple( + TransparencyWitnessKeyDeclaration( + witness_id=witness_id, + key_id=f"{witness_id}-key-1", + public_key_base64=base64.b64encode( + private_key.public_key().public_bytes_raw() + ).decode(), + ) + for witness_id, private_key in reversed(keys.items()) + ), + ) + return keys, build_transparency_witness_trust(declaration) + + +def _signed_checkpoint(proof, ledger): + keys, trust = _witness_trust() + template = build_witness_checkpoint_template(trust, proof, ledger) + signatures = tuple( + TransparencyWitnessSignature( + witness_id=request.witness_id, + key_id=f"{request.witness_id}-key-1", + signature_base64=base64.b64encode( + keys[request.witness_id].sign(request.statement.signing_bytes()) + ).decode(), + ) + for request in template.requests[:2] + ) + bundle = build_witness_checkpoint_bundle( + trust, + proof, + ledger, + signatures, + ) + return keys, trust, template, bundle + + +def test_rfc6962_compact_consistency_proofs_verify_for_non_power_of_two_trees() -> None: + authority_private, ledger = _authority_ledger() + for current_size in range(2, 13): + current_log = build_transparency_log("witness-log", _entries(current_size)) + current_head = _signed_head(current_log, authority_private, ledger) + for previous_size in range(1, current_size): + previous_log = build_transparency_log( + "witness-log", + current_log.entries[:previous_size], + ) + previous_head = _signed_head(previous_log, authority_private, ledger) + proof = build_transparency_consistency_proof( + current_log, + previous_tree_head=previous_head, + current_tree_head=current_head, + authority_root_ledger=ledger, + ) + + verify_transparency_consistency_proof(proof, ledger) + assert len(proof.audit_path_sha256) <= math.ceil( + math.log2(current_size) + ) + 1 + + +def test_consistency_path_matches_rfc6962_seven_leaf_example_shape() -> None: + _, _, current_log, proof = _proof(3, 7) + + def leaf(index: int) -> bytes: + return hashlib.sha256( + b"\x00" + current_log.entries[index].leaf_bytes() + ).digest() + + def node(left: bytes, right: bytes) -> bytes: + return hashlib.sha256(b"\x01" + left + right).digest() + + expected = ( + leaf(2), + leaf(3), + node(leaf(0), leaf(1)), + node(node(leaf(4), leaf(5)), leaf(6)), + ) + + assert proof.audit_path_sha256 == tuple(item.hex() for item in expected) + + +def test_consistency_proof_rejects_tampered_path_and_split_root() -> None: + _, ledger, _, proof = _proof(3, 7) + tampered = proof.model_copy( + update={"audit_path_sha256": ("00" * 32,) + proof.audit_path_sha256[1:]} + ) + tampered = tampered.model_copy( + update={ + "consistency_proof_sha256": _sha256( + tampered.model_dump( + mode="json", + exclude={"consistency_proof_sha256"}, + ) + ) + } + ) + + with pytest.raises(ValueError, match="consistency proof roots do not match"): + verify_transparency_consistency_proof(tampered, ledger) + + split_head = proof.current_tree_head.model_copy( + update={ + "statement": proof.current_tree_head.statement.model_copy( + update={"root_sha256": "ff" * 32} + ) + } + ) + split = proof.model_copy(update={"current_tree_head": split_head}) + with pytest.raises(ValueError, match="tree head fingerprint|signature"): + verify_transparency_consistency_proof(split, ledger) + + +def test_witness_trust_requires_intersecting_quorum() -> None: + keys, trust = _witness_trust() + + assert trust.minimum_distinct_witnesses == 2 + assert len(keys) == 3 + payload = trust.model_dump(mode="json", exclude={"witness_trust_sha256"}) + payload["minimum_distinct_witnesses"] = 1 + payload["keys"] = [ + { + "witness_id": key.witness_id, + "key_id": key.key_id, + "public_key_base64": key.public_key_base64, + "status": key.status, + } + for key in trust.keys + ] + with pytest.raises(ValueError, match="strict majority"): + build_transparency_witness_trust( + TransparencyWitnessTrustDeclaration.model_validate(payload) + ) + + +def test_witness_checkpoint_requires_distinct_valid_quorum_signatures() -> None: + _, ledger, _, proof = _proof(3, 7) + _, trust, template, bundle = _signed_checkpoint(proof, ledger) + + verify_witness_checkpoint_bundle(trust, proof, ledger, bundle) + + assert len(template.requests) == 3 + assert len(bundle.signatures) == 2 + with pytest.raises(ValueError, match="witness quorum"): + build_witness_checkpoint_bundle( + trust, + proof, + ledger, + bundle.signatures[:1], + ) + tampered_signature = bundle.signatures[0].model_copy( + update={"signature_base64": base64.b64encode(bytes(64)).decode()} + ) + tampered = bundle.model_copy( + update={"signatures": (tampered_signature,) + bundle.signatures[1:]} + ) + tampered = tampered.model_copy( + update={ + "witness_checkpoint_sha256": _sha256( + tampered.model_dump( + mode="json", + exclude={"witness_checkpoint_sha256"}, + ) + ) + } + ) + with pytest.raises(ValueError, match="witness signature is invalid"): + verify_witness_checkpoint_bundle(trust, proof, ledger, tampered) + + +def test_witness_checkpoints_detect_same_size_split_view() -> None: + authority_private, ledger = _authority_ledger() + common = _entries(3) + first_log = build_transparency_log("witness-log", common + _entries(4, offset=10)) + second_log = build_transparency_log("witness-log", common + _entries(4, offset=20)) + previous_log = build_transparency_log("witness-log", common) + previous_head = _signed_head(previous_log, authority_private, ledger) + + bundles: list[SignedWitnessCheckpoint] = [] + for current_log in (first_log, second_log): + proof = build_transparency_consistency_proof( + current_log, + previous_tree_head=previous_head, + current_tree_head=_signed_head(current_log, authority_private, ledger), + authority_root_ledger=ledger, + ) + bundles.append(_signed_checkpoint(proof, ledger)[3]) + + with pytest.raises(ValueError, match="split-view checkpoint"): + detect_witness_checkpoint_conflict(bundles[0], bundles[1]) + + +def test_consistency_and_witness_cli_paths_are_offline_and_private_key_free( + tmp_path: Path, +) -> None: + _, ledger, current_log, expected_proof = _proof(3, 7) + inputs = { + "current-log": current_log.to_json(), + "previous-tree-head": expected_proof.previous_tree_head.model_dump_json(), + "current-tree-head": expected_proof.current_tree_head.model_dump_json(), + "authority-root-ledger": ledger.to_json(), + } + paths: dict[str, Path] = {} + for name, payload in inputs.items(): + paths[name] = tmp_path / f"{name}.json" + paths[name].write_text(payload, encoding="utf-8") + before = {path: path.read_bytes() for path in paths.values()} + proof_path = tmp_path / "consistency-proof.json" + args = ["benchmark", "agent-cli-transparency-consistency-proof"] + for name, path in paths.items(): + args.extend([f"--{name}", str(path)]) + args.extend(["--output", str(proof_path), "--json"]) + + result = runner.invoke(app, args) + + assert result.exit_code == 0 + proof = TransparencyConsistencyProof.model_validate_json( + proof_path.read_text(encoding="utf-8") + ) + assert proof == expected_proof + assert before == {path: path.read_bytes() for path in paths.values()} + + _, trust = _witness_trust() + declaration_payload = { + "schema_version": trust.schema_version, + "log_id": trust.log_id, + "minimum_distinct_witnesses": trust.minimum_distinct_witnesses, + "keys": [ + { + "witness_id": key.witness_id, + "key_id": key.key_id, + "algorithm": key.algorithm, + "public_key_base64": key.public_key_base64, + "status": key.status, + } + for key in trust.keys + ], + } + declaration_path = tmp_path / "witness-declaration.json" + declaration_path.write_text( + json.dumps(declaration_payload, sort_keys=True), + encoding="utf-8", + ) + trust_path = tmp_path / "witness-trust.json" + trust_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-witness-trust", + "--declaration", + str(declaration_path), + "--output", + str(trust_path), + "--json", + ], + ) + assert trust_result.exit_code == 0 + assert json.loads(trust_result.output) == json.loads(trust.to_json()) + + template_path = tmp_path / "witness-template.json" + template_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli-witness-checkpoint-template", + "--witness-trust", + str(trust_path), + "--consistency-proof", + str(proof_path), + "--authority-root-ledger", + str(paths["authority-root-ledger"]), + "--output", + str(template_path), + "--json", + ], + ) + + assert template_result.exit_code == 0 + template = WitnessCheckpointTemplate.model_validate_json( + template_path.read_text(encoding="utf-8") + ) + assert len(template.requests) == 3 + assert "private" not in template.to_json().lower() + + +def test_committed_witness_trust_template_has_intersecting_quorum() -> None: + root = Path(__file__).parents[3] + declaration = TransparencyWitnessTrustDeclaration.model_validate_json( + ( + root / "benchmarks/templates/agent_cli_witness_trust.example.json" + ).read_text(encoding="utf-8") + ) + + trust = build_transparency_witness_trust(declaration) + + assert trust.log_id == "example-org-agent-cli" + assert trust.minimum_distinct_witnesses == 2 + assert len({key.witness_id for key in trust.keys}) == 3 diff --git a/tests/unit/domain/test_agent_engine_events.py b/tests/unit/domain/test_agent_engine_events.py new file mode 100644 index 0000000..6c6063b --- /dev/null +++ b/tests/unit/domain/test_agent_engine_events.py @@ -0,0 +1,35 @@ +"""Domain tests for normalized native agent engine events.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType +from domain.value_objects.agent_engine import AgentEngineType + + +def test_agent_engine_event_is_strict_immutable_and_serializable() -> None: + event = AgentEngineEvent( + type=AgentEngineEventType.TOOL_STARTED, + engine=AgentEngineType.CODEX_CLI, + sequence=2, + session_id="thread-1", + item_id="item-1", + item_type="command_execution", + text="pytest -q", + payload={"status": "in_progress"}, + ) + + assert event.model_dump(mode="json")["type"] == "tool_started" + with pytest.raises(ValidationError): + event.sequence = 3 + + +def test_agent_engine_event_rejects_negative_sequence() -> None: + with pytest.raises(ValidationError): + AgentEngineEvent( + type=AgentEngineEventType.RUN_STARTED, + engine=AgentEngineType.CODEX_CLI, + sequence=-1, + ) diff --git a/tests/unit/domain/test_chat_cli_domain.py b/tests/unit/domain/test_chat_cli_domain.py index d4dce96..c1c8d4b 100644 --- a/tests/unit/domain/test_chat_cli_domain.py +++ b/tests/unit/domain/test_chat_cli_domain.py @@ -17,6 +17,13 @@ from domain.entities.chat_event import ChatEvent, ChatEventType from domain.entities.chat_session import ChatSession, ChatSessionStatus, PermissionMode from domain.entities.council_runtime import CouncilDecision, CouncilRole, CouncilTurn +from domain.entities.hook import ( + HookDefinition, + HookDiagnostic, + HookExecutionRequest, + HookExecutionResult, + HookType, +) from domain.entities.workspace_context import ( ContextIndex, ContextSourceType, @@ -26,8 +33,11 @@ from domain.ports.context_discovery import ContextDiscoveryPort from domain.ports.council_runtime import CouncilRuntimePort from domain.ports.engine_registry import EngineProfile, EngineRegistryPort, EngineRuntimeKind +from domain.ports.hook_executor import HookExecutorPort +from domain.ports.hook_registry import HookRegistryPort from domain.ports.tool_executor import ToolExecutionRequest, ToolExecutionResult, ToolExecutorPort from domain.value_objects import RiskLevel +from domain.value_objects.agent_engine import AgentEngineType def test_chat_event_requires_ordered_append_only_fields() -> None: @@ -83,6 +93,31 @@ def test_chat_session_records_events_without_mutating_previous_state() -> None: assert updated.status is ChatSessionStatus.ACTIVE +def test_chat_session_tracks_native_session_provenance_from_engine_event() -> None: + session = ChatSession.start( + session_id="chat-1", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + session, _ = session.record_event( + ChatEventType.CONTEXT_INDEXED, + {"workspace_root": "/repo"}, + ) + + updated, _ = session.record_event( + ChatEventType.ENGINE_EVENT, + { + "engine": "codex_cli", + "session_id": "thread-1", + "type": "run_started", + }, + ) + + native = updated.native_sessions[AgentEngineType.CODEX_CLI.value] + assert native.session_id == "thread-1" + assert native.workspace_root == "/repo" + assert native.permission_mode is PermissionMode.WORKSPACE_WRITE + + def test_chat_session_close_marks_terminal_state() -> None: session = ChatSession.start( session_id="chat-1", @@ -187,6 +222,8 @@ def test_phase_1_ports_are_abstract_contracts() -> None: assert inspect.isabstract(CouncilRuntimePort) assert inspect.isabstract(ToolExecutorPort) assert inspect.isabstract(EngineRegistryPort) + assert inspect.isabstract(HookRegistryPort) + assert inspect.isabstract(HookExecutorPort) def test_tool_executor_contract_uses_risk_classification() -> None: @@ -225,3 +262,56 @@ def test_engine_registry_profile_models_runtime_capabilities() -> None: assert profile.id == "ollama" assert profile.kind is EngineRuntimeKind.LOCAL_MODEL assert "planning" in profile.capabilities + + +def test_hook_definition_tracks_type_and_source() -> None: + hook = HookDefinition( + name="lint", + hook_type=HookType.PRE_COMMIT, + command="uv run --extra dev ruff check .", + enabled=True, + source_path=".morphic/hooks/lint.json", + ) + diagnostic = HookDiagnostic( + name="Hook: lint", + status="OK", + message="pre_commit hook is valid", + source_path=hook.source_path, + ) + + assert hook.hook_type is HookType.PRE_COMMIT + assert hook.enabled + assert diagnostic.status == "OK" + + +def test_hook_execution_contract_tracks_request_and_result() -> None: + request = HookExecutionRequest( + session_id="chat-1", + hook_name="lint", + hook_type=HookType.PRE_COMMIT, + command="uv run --extra dev ruff check .", + source_path=".morphic/hooks/lint.json", + ) + result = HookExecutionResult( + request_id=request.id, + success=True, + stdout_summary="All checks passed", + stderr_summary="", + exit_code=0, + ) + + assert request.hook_name == "lint" + assert request.hook_type is HookType.PRE_COMMIT + assert result.request_id == request.id + assert result.success + + +def test_hook_execution_request_rejects_empty_command() -> None: + with pytest.raises(ValidationError): + HookExecutionRequest( + session_id="chat-1", + hook_name="lint", + hook_type=HookType.PRE_COMMIT, + command="", + source_path=".morphic/hooks/lint.json", + ) diff --git a/tests/unit/infrastructure/test_claude_code_driver.py b/tests/unit/infrastructure/test_claude_code_driver.py index 34a541f..897e904 100644 --- a/tests/unit/infrastructure/test_claude_code_driver.py +++ b/tests/unit/infrastructure/test_claude_code_driver.py @@ -7,12 +7,22 @@ import pytest +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType +from domain.entities.chat_session import PermissionMode from domain.ports.agent_engine import AgentEngineCapabilities, AgentEngineResult from domain.value_objects.agent_engine import AgentEngineType from infrastructure.agent_cli._subprocess_base import CLIResult from infrastructure.agent_cli.claude_code_driver import ClaudeCodeDriver +class _CollectingEventSink: + def __init__(self) -> None: + self.events: list[AgentEngineEvent] = [] + + async def publish(self, event: AgentEngineEvent) -> None: + self.events.append(event) + + @pytest.fixture() def driver(): return ClaudeCodeDriver(enabled=True, cli_path="claude") @@ -63,6 +73,104 @@ async def test_unavailable_when_binary_missing(self, driver): class TestRunTask: @pytest.mark.asyncio + async def test_scoped_stream_normalizes_claude_jsonl(self, driver): + lines = [ + json.dumps( + { + "type": "system", + "subtype": "init", + "session_id": "claude-session-1", + "model": "claude-sonnet-4-6", + } + ), + json.dumps( + { + "type": "assistant", + "session_id": "claude-session-1", + "message": { + "content": [ + { + "type": "tool_use", + "id": "tool-1", + "name": "Bash", + "input": {"command": "pytest -q"}, + } + ] + }, + } + ), + json.dumps( + { + "type": "user", + "session_id": "claude-session-1", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "tool-1", + "content": "3517 passed", + } + ] + }, + } + ), + json.dumps( + { + "type": "result", + "subtype": "success", + "session_id": "claude-session-1", + "result": "All tests pass.", + "total_cost_usd": 0.02, + } + ), + ] + + async def fake_stream(cmd, *, timeout, on_stdout_line, env=None, cwd=None): + del cmd, timeout, env, cwd + for line in lines: + await on_stdout_line(line) + return CLIResult(stdout="\n".join(lines), stderr="", returncode=0) + + sink = _CollectingEventSink() + with patch.object(driver, "_run_cli_streaming", side_effect=fake_stream): + result = await driver.run_task_scoped_stream( + "Fix tests", + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + ) + + assert result.output == "All tests pass." + assert result.metadata["session_id"] == "claude-session-1" + assert result.cost_usd == 0.02 + assert [event.type for event in sink.events] == [ + AgentEngineEventType.RUN_STARTED, + AgentEngineEventType.TOOL_STARTED, + AgentEngineEventType.TOOL_COMPLETED, + AgentEngineEventType.RUN_COMPLETED, + ] + + @pytest.mark.asyncio + async def test_resume_stream_uses_explicit_session_id(self, driver): + sink = _CollectingEventSink() + with patch.object( + driver, + "_run_cli_streaming", + return_value=CLIResult(stdout="{}", stderr="", returncode=0), + ) as mock_run: + await driver.resume_task_scoped_stream( + "Continue", + resume_session_id="claude-session-1", + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + ) + + cmd = mock_run.call_args[0][0] + assert cmd[cmd.index("--resume") + 1] == "claude-session-1" + assert cmd[cmd.index("--output-format") + 1] == "stream-json" + assert mock_run.call_args.kwargs["cwd"] == "/workspace" + @pytest.mark.asyncio async def test_valid_json_output(self, driver): json_output = json.dumps({"result": "Hello world", "session_id": "sess-123"}) with patch.object( @@ -146,13 +254,61 @@ async def test_command_shape(self, driver): "json", "--max-turns", "10", - "--setting-sources", - "user", - "--allowedTools", - "Bash,Read,Write,Edit,WebFetch,WebSearch", + "--permission-mode", + "plan", ] assert cmd == expected + @pytest.mark.asyncio + async def test_scoped_workspace_write_preserves_native_harness(self, driver): + with patch.object( + driver, + "_run_cli", + return_value=CLIResult(stdout="{}", stderr="", returncode=0), + ) as mock_run: + await driver.run_task_scoped( + "Fix tests", + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + + cmd = mock_run.call_args[0][0] + assert cmd[cmd.index("--permission-mode") + 1] == "acceptEdits" + assert mock_run.call_args.kwargs["cwd"] == "/workspace" + assert "--setting-sources" not in cmd + assert "--allowedTools" not in cmd + assert "--disable-slash-commands" not in cmd + + @pytest.mark.asyncio + async def test_danger_full_access_requires_explicit_bypass_flag(self, driver): + with patch.object( + driver, + "_run_cli", + return_value=CLIResult(stdout="{}", stderr="", returncode=0), + ) as mock_run: + await driver.run_task_scoped( + "Fix tests", + workspace_root="/workspace", + permission_mode=PermissionMode.DANGER_FULL_ACCESS, + ) + + cmd = mock_run.call_args[0][0] + assert "--dangerously-skip-permissions" in cmd + assert cmd[cmd.index("--permission-mode") + 1] == "bypassPermissions" + + @pytest.mark.asyncio + async def test_confirm_destructive_is_rejected_before_subprocess(self, driver): + with patch.object(driver, "_run_cli") as mock_run: + result = await driver.run_task_scoped( + "Fix tests", + workspace_root="/workspace", + permission_mode=PermissionMode.CONFIRM_DESTRUCTIVE, + ) + + assert result.success is False + assert "confirm-destructive" in str(result.error) + mock_run.assert_not_called() + @pytest.mark.asyncio async def test_model_override(self, driver): """Model flag is appended when specified.""" diff --git a/tests/unit/infrastructure/test_codex_cli_driver.py b/tests/unit/infrastructure/test_codex_cli_driver.py index 70c8a96..473ba3f 100644 --- a/tests/unit/infrastructure/test_codex_cli_driver.py +++ b/tests/unit/infrastructure/test_codex_cli_driver.py @@ -7,12 +7,22 @@ import pytest +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType +from domain.entities.chat_session import PermissionMode from domain.ports.agent_engine import AgentEngineCapabilities, AgentEngineResult from domain.value_objects.agent_engine import AgentEngineType from infrastructure.agent_cli._subprocess_base import CLIResult from infrastructure.agent_cli.codex_cli_driver import CodexCLIDriver +class _CollectingEventSink: + def __init__(self) -> None: + self.events: list[AgentEngineEvent] = [] + + async def publish(self, event: AgentEngineEvent) -> None: + self.events.append(event) + + @pytest.fixture() def driver(): return CodexCLIDriver(enabled=True, cli_path="codex") @@ -62,6 +72,127 @@ async def test_unavailable_when_binary_missing(self, driver): class TestRunTask: + @pytest.mark.asyncio + async def test_scoped_stream_publishes_jsonl_events_incrementally(self, driver): + lines = [ + json.dumps({"type": "thread.started", "thread_id": "thread-live"}), + json.dumps({"type": "turn.started"}), + json.dumps( + { + "type": "item.completed", + "item": { + "id": "message-1", + "type": "agent_message", + "text": "Streaming complete.", + }, + } + ), + json.dumps({"type": "turn.completed", "usage": {"input_tokens": 10}}), + ] + + async def fake_stream(cmd, *, timeout, on_stdout_line, env=None): + del cmd, timeout, env + for line in lines: + await on_stdout_line(line) + return CLIResult(stdout="\n".join(lines), stderr="", returncode=0) + + sink = _CollectingEventSink() + with patch.object(driver, "_run_cli_streaming", side_effect=fake_stream): + result = await driver.run_task_scoped_stream( + "Fix tests", + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + ) + + assert result.success is True + assert result.output == "Streaming complete." + assert [event.type for event in sink.events] == [ + AgentEngineEventType.RUN_STARTED, + AgentEngineEventType.TURN_STARTED, + AgentEngineEventType.ASSISTANT_MESSAGE, + AgentEngineEventType.RUN_COMPLETED, + ] + assert [event.sequence for event in sink.events] == [0, 1, 2, 3] + assert all(event.session_id == "thread-live" for event in sink.events) + + @pytest.mark.asyncio + async def test_jsonl_output_is_normalized_and_final_message_selected(self, driver): + json_output = "\n".join( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-1"}), + json.dumps({"type": "turn.started"}), + json.dumps( + { + "type": "item.started", + "item": { + "id": "item-1", + "type": "command_execution", + "command": "pytest -q", + "status": "in_progress", + }, + } + ), + json.dumps( + { + "type": "item.completed", + "item": { + "id": "item-1", + "type": "command_execution", + "command": "pytest -q", + "status": "completed", + "exit_code": 0, + }, + } + ), + json.dumps( + { + "type": "item.completed", + "item": { + "id": "item-2", + "type": "agent_message", + "text": "Implemented the fix.", + }, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 80, + "output_tokens": 20, + "reasoning_output_tokens": 5, + }, + } + ), + ] + ) + with patch.object( + driver, + "_run_cli", + return_value=CLIResult(stdout=json_output, stderr="", returncode=0), + ): + result = await driver.run_task("Fix tests") + + assert result.success is True + assert result.output == "Implemented the fix." + assert result.metadata["session_id"] == "thread-1" + assert result.metadata["usage"] == { + "input_tokens": 100, + "cached_input_tokens": 80, + "output_tokens": 20, + "reasoning_output_tokens": 5, + } + assert [event["type"] for event in result.metadata["events"]] == [ + AgentEngineEventType.RUN_STARTED.value, + AgentEngineEventType.TURN_STARTED.value, + AgentEngineEventType.TOOL_STARTED.value, + AgentEngineEventType.TOOL_COMPLETED.value, + AgentEngineEventType.ASSISTANT_MESSAGE.value, + AgentEngineEventType.RUN_COMPLETED.value, + ] + @pytest.mark.asyncio async def test_valid_json_output(self, driver): json_output = json.dumps({"result": "Generated code"}) @@ -122,9 +253,117 @@ async def test_command_shape(self, driver): "_run_cli", return_value=CLIResult(stdout="{}", stderr="", returncode=0), ) as mock_run: - await driver.run_task("Build feature") + await driver.run_task_scoped( + "Build feature", + workspace_root="/workspace", + permission_mode=PermissionMode.READ_ONLY, + ) cmd = mock_run.call_args[0][0] - assert cmd == ["codex", "exec", "--json", "--full-auto", "Build feature"] + assert cmd == [ + "codex", + "exec", + "--json", + "--sandbox", + "read-only", + "--cd", + "/workspace", + "Build feature", + ] + + @pytest.mark.asyncio + async def test_resume_command_preserves_sandbox_and_workspace(self, driver): + sink = _CollectingEventSink() + with patch.object( + driver, + "_run_cli_streaming", + return_value=CLIResult(stdout="{}", stderr="", returncode=0), + ) as mock_run: + await driver.resume_task_scoped_stream( + "Continue fixing tests", + resume_session_id="thread-1", + workspace_root="/workspace", + permission_mode=PermissionMode.WORKSPACE_WRITE, + event_sink=sink, + ) + + assert mock_run.call_args[0][0] == [ + "codex", + "exec", + "--json", + "--sandbox", + "workspace-write", + "--cd", + "/workspace", + "resume", + "thread-1", + "Continue fixing tests", + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("permission_mode", "sandbox"), + [ + (PermissionMode.READ_ONLY, "read-only"), + (PermissionMode.WORKSPACE_WRITE, "workspace-write"), + (PermissionMode.DANGER_FULL_ACCESS, "danger-full-access"), + ], + ) + async def test_permission_mode_maps_to_explicit_sandbox( + self, + driver, + permission_mode, + sandbox, + ): + with patch.object( + driver, + "_run_cli", + return_value=CLIResult(stdout="{}", stderr="", returncode=0), + ) as mock_run: + await driver.run_task_scoped( + "test", + workspace_root="/workspace", + permission_mode=permission_mode, + ) + + cmd = mock_run.call_args[0][0] + assert cmd[cmd.index("--sandbox") + 1] == sandbox + assert "--full-auto" not in cmd + + @pytest.mark.asyncio + async def test_confirm_destructive_is_rejected_before_subprocess(self, driver): + with patch.object(driver, "_run_cli") as mock_run: + result = await driver.run_task_scoped( + "test", + workspace_root="/workspace", + permission_mode=PermissionMode.CONFIRM_DESTRUCTIVE, + ) + + assert result.success is False + assert "confirm-destructive" in str(result.error) + mock_run.assert_not_called() + + @pytest.mark.asyncio + async def test_malformed_jsonl_line_does_not_hide_valid_final_message(self, driver): + stdout = "\n".join( + [ + "not-json", + json.dumps( + { + "type": "item.completed", + "item": {"id": "a", "type": "agent_message", "text": "done"}, + } + ), + ] + ) + with patch.object( + driver, + "_run_cli", + return_value=CLIResult(stdout=stdout, stderr="", returncode=0), + ): + result = await driver.run_task("test") + + assert result.output == "done" + assert result.metadata["parse_errors"] == 1 @pytest.mark.asyncio async def test_model_override(self, driver): @@ -208,3 +447,4 @@ def test_returns_capabilities(self, driver): assert caps.engine_type == AgentEngineType.CODEX_CLI assert caps.supports_sandbox is True assert caps.supports_mcp is True + assert caps.supports_streaming is True diff --git a/tests/unit/infrastructure/test_noop_hook_executor.py b/tests/unit/infrastructure/test_noop_hook_executor.py new file mode 100644 index 0000000..f20aabb --- /dev/null +++ b/tests/unit/infrastructure/test_noop_hook_executor.py @@ -0,0 +1,27 @@ +"""Tests for the no-op chat hook executor adapter.""" + +from __future__ import annotations + +import pytest + +from domain.entities.hook import HookExecutionRequest, HookType +from infrastructure.hooks.noop_hook_executor import NoopHookExecutor + + +@pytest.mark.asyncio +async def test_noop_hook_executor_returns_success_without_running_command() -> None: + executor = NoopHookExecutor() + request = HookExecutionRequest( + session_id="chat-1", + hook_name="pre-log", + hook_type=HookType.PRE_TOOL, + command="exit 99", + source_path=".morphic/hooks/pre-log.json", + ) + + result = await executor.execute(request) + + assert result.request_id == request.id + assert result.success + assert result.exit_code == 0 + assert "not executed" in result.stdout_summary diff --git a/tests/unit/infrastructure/test_noop_tool_executor.py b/tests/unit/infrastructure/test_noop_tool_executor.py new file mode 100644 index 0000000..a27cd8c --- /dev/null +++ b/tests/unit/infrastructure/test_noop_tool_executor.py @@ -0,0 +1,25 @@ +"""Tests for the no-op chat tool executor adapter.""" + +from __future__ import annotations + +import pytest + +from domain.ports.tool_executor import ToolExecutionRequest +from infrastructure.tools.noop_tool_executor import NoopToolExecutor + + +@pytest.mark.asyncio +async def test_noop_tool_executor_returns_success_without_running_tool() -> None: + executor = NoopToolExecutor() + request = ToolExecutionRequest( + session_id="chat-1", + tool_name="fs_write", + arguments={"path": "x.txt", "content": "x"}, + ) + + result = await executor.execute(request) + + assert result.request_id == request.id + assert result.success + assert result.exit_code == 0 + assert "not executed" in result.stdout_summary diff --git a/tests/unit/infrastructure/test_route_chat_direct_runtime.py b/tests/unit/infrastructure/test_route_chat_direct_runtime.py new file mode 100644 index 0000000..ebed084 --- /dev/null +++ b/tests/unit/infrastructure/test_route_chat_direct_runtime.py @@ -0,0 +1,314 @@ +"""Single-engine route-backed runtime tests for Morphic Chat CLI.""" + +from __future__ import annotations + +import pytest + +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType +from domain.entities.chat_event import ChatEventType +from domain.entities.chat_session import ChatSession, PermissionMode +from domain.entities.council_runtime import CouncilRole +from domain.entities.workspace_context import ( + ContextIndex, + ContextSourceType, + WorkspaceContextSource, +) +from domain.ports.agent_engine import AgentEngineResult +from domain.value_objects.agent_engine import AgentEngineType +from domain.value_objects.model_tier import TaskType +from infrastructure.council.route_chat_direct_runtime import RouteChatDirectRuntime + +pytestmark = pytest.mark.asyncio + + +class _FakeRouteToEngine: + def __init__(self, result: AgentEngineResult) -> None: + self._result = result + self.calls: list[dict[str, object]] = [] + + async def execute(self, **kwargs: object) -> AgentEngineResult: + self.calls.append(kwargs) + return self._result + + +class _CollectingEventSink: + def __init__(self) -> None: + self.events: list[AgentEngineEvent] = [] + + async def publish(self, event: AgentEngineEvent) -> None: + self.events.append(event) + + +def _session( + permission_mode: PermissionMode = PermissionMode.DANGER_FULL_ACCESS, +) -> ChatSession: + return ChatSession.start( + session_id="chat-direct-1", + goal="Fix the failing tests", + permission_mode=permission_mode, + ) + + +def _context() -> ContextIndex: + return ContextIndex( + workspace_root="/workspace", + sources=[ + WorkspaceContextSource( + source_path="AGENTS.md", + source_type=ContextSourceType.AGENTS_MD, + scope="root", + precedence=1000, + content_hash="sha256:test", + sections=["Rules"], + ) + ], + ) + + +async def test_direct_runtime_makes_one_route_call_and_maps_result() -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=True, + output="Implemented the fix and ran tests.", + cost_usd=0.04, + duration_seconds=2.5, + metadata={ + "events": [ + { + "type": "run_started", + "engine": "codex_cli", + "sequence": 0, + "session_id": "thread-1", + "item_id": None, + "item_type": None, + "text": None, + "payload": { + "type": "thread.started", + "thread_id": "thread-1", + }, + } + ] + }, + ) + ) + + turns, decision = await RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CODEX_CLI, + ).deliberate( + session=_session(), + context=_context(), + user_message="Fix the failing tests", + ) + + assert len(route.calls) == 1 + assert route.calls[0]["task"] == "Fix the failing tests" + assert route.calls[0]["task_type"] is TaskType.CODE_GENERATION + assert route.calls[0]["preferred_engine"] is AgentEngineType.CODEX_CLI + assert route.calls[0]["context"] == "context_sources=1; sources=AGENTS.md" + assert route.calls[0]["workspace_root"] == "/workspace" + assert route.calls[0]["permission_mode"] is PermissionMode.DANGER_FULL_ACCESS + + assert len(turns) == 1 + assert turns[0].role is CouncilRole.IMPLEMENTER + assert turns[0].engine_id == "codex_cli" + assert turns[0].content == "Implemented the fix and ran tests." + assert turns[0].cost_usd == 0.04 + assert turns[0].latency_ms == 2500 + assert len(turns[0].engine_events) == 1 + assert turns[0].engine_events[0].type is AgentEngineEventType.RUN_STARTED + assert turns[0].engine_events[0].session_id == "thread-1" + assert decision.leader_engine_id == "codex_cli" + assert decision.selected_role is CouncilRole.IMPLEMENTER + assert decision.selected_content == "Implemented the fix and ran tests." + + +async def test_direct_runtime_passes_event_sink_to_streaming_route() -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=True, + output="Done.", + ) + ) + sink = _CollectingEventSink() + + await RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CODEX_CLI, + ).deliberate_stream( + session=_session(PermissionMode.WORKSPACE_WRITE), + context=_context(), + user_message="Fix tests", + event_sink=sink, + ) + + assert route.calls[0]["event_sink"] is sink + + +async def test_direct_runtime_resumes_matching_native_session() -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=True, + output="Continued.", + ) + ) + session = ChatSession.start( + session_id="chat-direct-1", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + session, _ = session.record_event( + ChatEventType.CONTEXT_INDEXED, + {"workspace_root": "/workspace"}, + ) + session, _ = session.record_event( + ChatEventType.ENGINE_EVENT, + {"engine": "codex_cli", "session_id": "thread-1"}, + ) + + await RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CODEX_CLI, + ).deliberate_stream( + session=session, + context=_context(), + user_message="continue", + event_sink=_CollectingEventSink(), + ) + + assert route.calls[0]["resume_session_id"] == "thread-1" + assert route.calls[0]["resume_engine"] is AgentEngineType.CODEX_CLI + + +async def test_direct_runtime_rejects_resume_in_different_workspace() -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=True, + output="must not run", + ) + ) + session = ChatSession.start( + session_id="chat-direct-1", + permission_mode=PermissionMode.WORKSPACE_WRITE, + ) + session, _ = session.record_event( + ChatEventType.CONTEXT_INDEXED, + {"workspace_root": "/original"}, + ) + session, _ = session.record_event( + ChatEventType.ENGINE_EVENT, + {"engine": "codex_cli", "session_id": "thread-1"}, + ) + + with pytest.raises(PermissionError, match="workspace"): + await RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CODEX_CLI, + ).deliberate_stream( + session=session, + context=_context(), + user_message="continue", + event_sink=_CollectingEventSink(), + ) + + assert route.calls == [] + + +@pytest.mark.parametrize( + ("success", "output", "error", "message"), + [ + (False, "", "engine unavailable", "engine unavailable"), + (True, " ", None, "returned no output"), + ], +) +async def test_direct_runtime_reports_route_failure_without_local_fallback( + success: bool, + output: str, + error: str | None, + message: str, +) -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=success, + output=output, + error=error, + ) + ) + + with pytest.raises(RuntimeError, match=message): + await RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CODEX_CLI, + ).deliberate( + session=_session(), + context=_context(), + user_message="Fix the failing tests", + ) + + assert len(route.calls) == 1 + + +async def test_direct_runtime_rejects_confirm_destructive_without_prompt_channel() -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=True, + output="Would have edited files", + ) + ) + + with pytest.raises(PermissionError, match="confirm-destructive"): + await RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CODEX_CLI, + ).deliberate( + session=_session(PermissionMode.CONFIRM_DESTRUCTIVE), + context=_context(), + user_message="Fix the failing tests", + ) + + assert route.calls == [] + + +async def test_direct_runtime_accepts_workspace_write_for_codex() -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=True, + output="Edited workspace files", + ) + ) + + await RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CODEX_CLI, + ).deliberate( + session=_session(PermissionMode.WORKSPACE_WRITE), + context=_context(), + user_message="Fix tests", + ) + + assert route.calls[0]["permission_mode"] is PermissionMode.WORKSPACE_WRITE + + +async def test_direct_runtime_requires_supported_explicit_native_preference( +) -> None: + route = _FakeRouteToEngine( + AgentEngineResult( + engine=AgentEngineType.CODEX_CLI, + success=True, + output="unused", + ) + ) + + with pytest.raises(ValueError, match="codex_cli"): + RouteChatDirectRuntime(route) + runtime = RouteChatDirectRuntime( + route, + preferred_engine=AgentEngineType.CLAUDE_CODE, + ) + assert runtime._preferred_engine is AgentEngineType.CLAUDE_CODE diff --git a/tests/unit/infrastructure/test_shell_hook_executor.py b/tests/unit/infrastructure/test_shell_hook_executor.py new file mode 100644 index 0000000..b4c0caf --- /dev/null +++ b/tests/unit/infrastructure/test_shell_hook_executor.py @@ -0,0 +1,85 @@ +"""Tests for shell-backed chat hook execution.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from domain.entities.execution import Action, Observation +from domain.entities.hook import HookExecutionRequest, HookType +from domain.ports.local_executor import LocalExecutorPort +from domain.value_objects.status import ObservationStatus +from infrastructure.hooks.shell_hook_executor import ShellHookExecutor + + +class FakeLocalExecutor(LocalExecutorPort): + def __init__(self, *, status: ObservationStatus, result: str) -> None: + self.actions: list[Action] = [] + self._status = status + self._result = result + + async def execute(self, action: Action) -> Observation: + self.actions.append(action) + return Observation(status=self._status, result=self._result) + + async def undo_last(self) -> Observation: + return Observation(status=ObservationStatus.SUCCESS, result="undone") + + async def get_undo_stack_size(self) -> int: + return 0 + + +def _request() -> HookExecutionRequest: + return HookExecutionRequest( + session_id="chat-1", + hook_name="lint", + hook_type=HookType.PRE_TOOL, + command="uv run --extra dev ruff check .", + source_path=".morphic/hooks/lint.json", + ) + + +@pytest.mark.asyncio +async def test_shell_hook_executor_maps_hook_to_laee_shell_exec(tmp_path: Path) -> None: + local_executor = FakeLocalExecutor( + status=ObservationStatus.SUCCESS, + result="All checks passed", + ) + + result = await ShellHookExecutor( + local_executor=local_executor, + workspace_root=tmp_path, + timeout_seconds=12, + ).execute(_request()) + + assert result.success + assert result.stdout_summary == "All checks passed" + assert result.exit_code == 0 + assert local_executor.actions[0].tool == "shell_exec" + assert local_executor.actions[0].args == { + "cmd": "uv run --extra dev ruff check .", + "cwd": str(tmp_path), + "timeout": 12, + } + assert "hook lint" in local_executor.actions[0].description + + +@pytest.mark.asyncio +async def test_shell_hook_executor_maps_denied_observation_to_failed_result( + tmp_path: Path, +) -> None: + local_executor = FakeLocalExecutor( + status=ObservationStatus.DENIED, + result="Action requires approval", + ) + + result = await ShellHookExecutor( + local_executor=local_executor, + workspace_root=tmp_path, + ).execute(_request()) + + assert result.success is False + assert result.stdout_summary == "" + assert result.stderr_summary == "Action requires approval" + assert result.exit_code == 1 diff --git a/tests/unit/infrastructure/test_subprocess_base.py b/tests/unit/infrastructure/test_subprocess_base.py new file mode 100644 index 0000000..e05f358 --- /dev/null +++ b/tests/unit/infrastructure/test_subprocess_base.py @@ -0,0 +1,105 @@ +"""Tests for incremental subprocess output delivery.""" + +from __future__ import annotations + +import asyncio +import sys +from unittest.mock import patch + +import pytest + +from infrastructure.agent_cli._subprocess_base import SubprocessMixin + + +class _BlockingReader: + async def readline(self) -> bytes: + await asyncio.Future() + return b"" + + async def read(self) -> bytes: + await asyncio.Future() + return b"" + + +class _CancellableProcess: + def __init__(self) -> None: + self.stdout = _BlockingReader() + self.stderr = _BlockingReader() + self.returncode: int | None = None + self.terminated = False + self.killed = False + self._stopped = asyncio.Event() + + async def communicate(self): + await self._stopped.wait() + return b"", b"" + + async def wait(self) -> int: + await self._stopped.wait() + return self.returncode or 0 + + def terminate(self) -> None: + self.terminated = True + self.returncode = -15 + self._stopped.set() + + def kill(self) -> None: + self.killed = True + self.returncode = -9 + self._stopped.set() + + +@pytest.mark.asyncio +async def test_run_cli_streaming_delivers_stdout_lines_and_collects_stderr() -> None: + runner = SubprocessMixin() + delivered: list[str] = [] + + async def on_stdout_line(line: str) -> None: + delivered.append(line) + + result = await runner._run_cli_streaming( + [ + sys.executable, + "-c", + ( + "import sys; " + "print('first', flush=True); " + "print('second', flush=True); " + "print('warning', file=sys.stderr, flush=True)" + ), + ], + timeout=5.0, + on_stdout_line=on_stdout_line, + ) + + assert result.returncode == 0 + assert delivered == ["first", "second"] + assert result.stdout == "first\nsecond\n" + assert result.stderr == "warning\n" + + +@pytest.mark.asyncio +async def test_run_cli_streaming_terminates_process_and_propagates_cancellation() -> None: + runner = SubprocessMixin() + process = _CancellableProcess() + + async def ignore_line(line: str) -> None: + del line + + with patch( + "infrastructure.agent_cli._subprocess_base.asyncio.create_subprocess_exec", + return_value=process, + ): + task = asyncio.create_task( + runner._run_cli_streaming( + ["codex", "exec"], + on_stdout_line=ignore_line, + ) + ) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert process.terminated is True + assert process.killed is False diff --git a/tests/unit/interface/test_benchmark_cli.py b/tests/unit/interface/test_benchmark_cli.py index a4faae7..ad9b6bd 100644 --- a/tests/unit/interface/test_benchmark_cli.py +++ b/tests/unit/interface/test_benchmark_cli.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from unittest.mock import MagicMock import pytest @@ -63,6 +64,116 @@ def test_benchmark_dedup(self) -> None: def test_benchmark_help(self) -> None: result = runner.invoke(app, ["benchmark", "--help"]) assert result.exit_code == 0 + assert "agent-cli" in result.output assert "continuity" in result.output assert "dedup" in result.output assert "run" in result.output + + def test_agent_cli_evaluates_recorded_results_as_json(self, tmp_path: object) -> None: + from pathlib import Path + + root = Path(str(tmp_path)) + manifest = root / "manifest.json" + results = root / "results.json" + manifest.write_text( + json.dumps( + { + "schema_version": 1, + "benchmark_id": "cli-001", + "task": { + "id": "task-001", + "goal": "Apply one identical change", + "workspace_revision": "abc123", + "checks": ["unit"], + "handoff_assertions": ["decision"], + }, + "arms": ["codex_cli", "claude_code", "morphic_control"], + "repetitions": 1, + } + ), + encoding="utf-8", + ) + results.write_text( + json.dumps( + { + "schema_version": 1, + "benchmark_id": "cli-001", + "task_id": "task-001", + "observations": [ + { + "arm": arm, + "trial": 1, + "completed": True, + "accepted_patch": True, + "passed_checks": ["unit"], + "elapsed_seconds": 10.0, + "cost_usd": 0.1, + "human_interventions": 0, + "recovery_attempted": False, + "recovery_succeeded": False, + "passed_handoff_assertions": ["decision"], + } + for arm in ("codex_cli", "claude_code", "morphic_control") + ], + } + ), + encoding="utf-8", + ) + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli", + "--manifest", + str(manifest), + "--results", + str(results), + "--json", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["benchmark_id"] == "cli-001" + assert payload["observation_count"] == 3 + assert "overall_score" not in payload + + human_result = runner.invoke( + app, + [ + "benchmark", + "agent-cli", + "--manifest", + str(manifest), + "--results", + str(results), + ], + ) + assert human_result.exit_code == 0 + assert "Metric leaders" in human_result.output + assert "completion_rate" in human_result.output + + def test_agent_cli_reports_invalid_recorded_results(self, tmp_path: object) -> None: + from pathlib import Path + + root = Path(str(tmp_path)) + manifest = root / "manifest.json" + results = root / "results.json" + manifest.write_text("{}", encoding="utf-8") + results.write_text("{}", encoding="utf-8") + + result = runner.invoke( + app, + [ + "benchmark", + "agent-cli", + "--manifest", + str(manifest), + "--results", + str(results), + ], + ) + + assert result.exit_code == 1 + assert "Invalid agent CLI benchmark input" in result.output diff --git a/tests/unit/interface/test_chat_cli.py b/tests/unit/interface/test_chat_cli.py index 33295a4..cb4d0dd 100644 --- a/tests/unit/interface/test_chat_cli.py +++ b/tests/unit/interface/test_chat_cli.py @@ -2,24 +2,39 @@ from __future__ import annotations +import asyncio import json from pathlib import Path import pytest from typer.testing import CliRunner +from domain.entities.agent_engine_event import AgentEngineEvent, AgentEngineEventType from domain.entities.approval import ApprovalRequest from domain.entities.chat_session import ChatSession from domain.entities.council_runtime import CouncilDecision, CouncilRole, CouncilTurn +from domain.entities.execution import Action, Observation from domain.entities.workspace_context import ContextIndex +from domain.ports.agent_engine import AgentEngineEventSinkPort +from domain.ports.council_runtime import StreamingCouncilRuntimePort from domain.ports.engine_registry import EngineProfile, EngineRuntimeKind +from domain.ports.local_executor import LocalExecutorPort from domain.value_objects import RiskLevel from domain.value_objects.agent_engine import AgentEngineType -from interface.cli.chat_command import _chat_doctor_payload, _role_engine_preferences +from domain.value_objects.status import ObservationStatus +from interface.cli.chat_command import ( + _chat_doctor_payload, + _chat_hook_executor, + _chat_tool_executor, + _role_engine_preferences, +) +from interface.cli.chat_control_transport import send_chat_control_command from interface.cli.chat_repl import ChatRepl from interface.cli.main import app +from interface.cli.native_event_progress import NativeEventProgressRenderer from interface.cli.renderers import render_approval_request from interface.cli.slash_commands import parse_slash_command +from interface.cli.turn_control import ActiveTurnController runner = CliRunner() @@ -63,6 +78,167 @@ async def deliberate( return [turn], decision +class _FakeReceiptCouncilRuntime: + async def deliberate( + self, + session: ChatSession, + context: ContextIndex, + user_message: str, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + del session, context + turn = CouncilTurn( + role=CouncilRole.IMPLEMENTER, + engine_id="codex_cli", + content=f"Completed: {user_message}", + cost_usd=0.125, + engine_events=[ + AgentEngineEvent( + type=AgentEngineEventType.RUN_COMPLETED, + engine=AgentEngineType.CODEX_CLI, + sequence=0, + payload={ + "type": "turn.completed", + "usage": {"input_tokens": 10, "output_tokens": 4}, + }, + ) + ], + ) + return [turn], CouncilDecision( + leader_engine_id="codex_cli", + selected_role=CouncilRole.IMPLEMENTER, + selected_content=turn.content, + rationale="receipt test", + ) + + +class _FakeStreamingCouncilRuntime(StreamingCouncilRuntimePort): + async def deliberate(self, session, context, user_message): + raise AssertionError("streaming path expected") + + async def deliberate_stream( + self, + session, + context, + user_message, + event_sink: AgentEngineEventSinkPort, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + del session, context + await event_sink.publish( + AgentEngineEvent( + type=AgentEngineEventType.RUN_STARTED, + engine=AgentEngineType.CODEX_CLI, + sequence=0, + session_id="thread-ui", + payload={"type": "thread.started"}, + ) + ) + turn = CouncilTurn( + role=CouncilRole.IMPLEMENTER, + engine_id="codex_cli", + content=f"Completed: {user_message}", + ) + return [turn], CouncilDecision( + leader_engine_id="codex_cli", + selected_role=CouncilRole.IMPLEMENTER, + selected_content=turn.content, + rationale="stream completed", + ) + + +class _BlockingStreamingCouncilRuntime(StreamingCouncilRuntimePort): + def __init__(self) -> None: + self.started = asyncio.Event() + + async def deliberate(self, session, context, user_message): + raise AssertionError("streaming path expected") + + async def deliberate_stream( + self, + session, + context, + user_message, + event_sink: AgentEngineEventSinkPort, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + del session, context, user_message + await event_sink.publish( + AgentEngineEvent( + type=AgentEngineEventType.RUN_STARTED, + engine=AgentEngineType.CODEX_CLI, + sequence=0, + session_id="thread-controlled", + payload={"type": "thread.started"}, + ) + ) + self.started.set() + await asyncio.Event().wait() + raise AssertionError("cancelled turn must not complete") + + +class _SteerableStreamingCouncilRuntime(StreamingCouncilRuntimePort): + def __init__(self) -> None: + self.first_turn_started = asyncio.Event() + self.messages: list[str] = [] + self.resumed_native_session_id: str | None = None + + async def deliberate(self, session, context, user_message): + raise AssertionError("streaming path expected") + + async def deliberate_stream( + self, + session, + context, + user_message, + event_sink: AgentEngineEventSinkPort, + ) -> tuple[list[CouncilTurn], CouncilDecision]: + del context + self.messages.append(user_message) + if len(self.messages) == 1: + await event_sink.publish( + AgentEngineEvent( + type=AgentEngineEventType.RUN_STARTED, + engine=AgentEngineType.CODEX_CLI, + sequence=0, + session_id="thread-steered", + payload={"type": "thread.started"}, + ) + ) + self.first_turn_started.set() + await asyncio.Event().wait() + raise AssertionError("steered turn must not complete") + + self.resumed_native_session_id = session.native_sessions["codex_cli"].session_id + turn = CouncilTurn( + role=CouncilRole.IMPLEMENTER, + engine_id="codex_cli", + content=f"Steered: {user_message}", + ) + return [turn], CouncilDecision( + leader_engine_id="codex_cli", + selected_role=CouncilRole.IMPLEMENTER, + selected_content=turn.content, + rationale="steered native session continued", + ) + + +class _CollectingEventSink(AgentEngineEventSinkPort): + def __init__(self) -> None: + self.events: list[AgentEngineEvent] = [] + + async def publish(self, event: AgentEngineEvent) -> None: + self.events.append(event) + + +class _FakeLocalExecutor(LocalExecutorPort): + async def execute(self, action: Action) -> Observation: + return Observation(status=ObservationStatus.SUCCESS, result="ok") + + async def undo_last(self) -> Observation: + return Observation(status=ObservationStatus.SUCCESS, result="undone") + + async def get_undo_stack_size(self) -> int: + return 0 + + def test_parse_slash_command_name_and_args() -> None: command = parse_slash_command("/resume latest") @@ -85,9 +261,121 @@ def test_chat_doctor_json_lists_engines() -> None: assert result.exit_code == 0 payload = json.loads(result.output) assert "engines" in payload + assert payload["hook_execution_mode"] == "noop" + assert payload["tool_execution_mode"] == "noop" assert any(engine["id"] == "ollama" for engine in payload["engines"]) +def test_chat_hook_executor_defaults_to_noop(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + from infrastructure.hooks.noop_hook_executor import NoopHookExecutor + + monkeypatch.delenv("MORPHIC_CHAT_HOOK_EXECUTION", raising=False) + + executor = _chat_hook_executor( + workspace_root=tmp_path, + local_executor_factory=lambda: _FakeLocalExecutor(), + ) + + assert isinstance(executor, NoopHookExecutor) + + +@pytest.mark.asyncio +async def test_chat_doctor_payload_reports_shell_hook_execution_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MORPHIC_CHAT_HOOK_EXECUTION", "shell") + + payload = await _chat_doctor_payload(engine_registry=_FakeEngineRegistry()) + + assert payload["hook_execution_mode"] == "shell" + + +def test_chat_hook_executor_uses_shell_only_with_explicit_opt_in( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + from infrastructure.hooks.shell_hook_executor import ShellHookExecutor + + monkeypatch.setenv("MORPHIC_CHAT_HOOK_EXECUTION", "shell") + + executor = _chat_hook_executor( + workspace_root=tmp_path, + local_executor_factory=lambda: _FakeLocalExecutor(), + ) + + assert isinstance(executor, ShellHookExecutor) + + +def test_chat_hook_executor_rejects_unknown_mode( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + monkeypatch.setenv("MORPHIC_CHAT_HOOK_EXECUTION", "always") + + with pytest.raises(ValueError) as exc_info: + _chat_hook_executor( + workspace_root=tmp_path, + local_executor_factory=lambda: _FakeLocalExecutor(), + ) + + assert "Invalid hook execution mode" in str(exc_info.value) + + +def test_chat_tool_executor_defaults_to_noop(monkeypatch: pytest.MonkeyPatch) -> None: + from infrastructure.tools.noop_tool_executor import NoopToolExecutor + + monkeypatch.delenv("MORPHIC_CHAT_TOOL_EXECUTION", raising=False) + + executor = _chat_tool_executor(local_executor_factory=lambda: _FakeLocalExecutor()) + + assert isinstance(executor, NoopToolExecutor) + + +def test_chat_tool_executor_uses_laee_only_with_explicit_opt_in( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from infrastructure.tools.laee_tool_executor import LaeeToolExecutor + + monkeypatch.setenv("MORPHIC_CHAT_TOOL_EXECUTION", "laee") + + executor = _chat_tool_executor(local_executor_factory=lambda: _FakeLocalExecutor()) + + assert isinstance(executor, LaeeToolExecutor) + + +def test_chat_tool_executor_rejects_unknown_mode(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MORPHIC_CHAT_TOOL_EXECUTION", "always") + + with pytest.raises(ValueError) as exc_info: + _chat_tool_executor(local_executor_factory=lambda: _FakeLocalExecutor()) + + assert "Invalid tool execution mode" in str(exc_info.value) + + +def test_chat_doctor_invalid_hook_execution_mode_exits_with_diagnostic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MORPHIC_CHAT_HOOK_EXECUTION", "always") + + with runner.isolated_filesystem(): + result = runner.invoke(app, ["chat", "--doctor", "--json"]) + + assert result.exit_code == 2 + assert "Invalid hook execution mode" in result.output + + +def test_chat_doctor_invalid_tool_execution_mode_exits_with_diagnostic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MORPHIC_CHAT_TOOL_EXECUTION", "always") + + with runner.isolated_filesystem(): + result = runner.invoke(app, ["chat", "--doctor", "--json"]) + + assert result.exit_code == 2 + assert "Invalid tool execution mode" in result.output + + @pytest.mark.asyncio async def test_chat_doctor_payload_uses_injected_engine_registry() -> None: payload = await _chat_doctor_payload(engine_registry=_FakeEngineRegistry()) @@ -110,6 +398,17 @@ async def test_chat_doctor_payload_uses_injected_engine_registry() -> None: ] +@pytest.mark.asyncio +async def test_chat_doctor_payload_reports_laee_tool_execution_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MORPHIC_CHAT_TOOL_EXECUTION", "laee") + + payload = await _chat_doctor_payload(engine_registry=_FakeEngineRegistry()) + + assert payload["tool_execution_mode"] == "laee" + + def test_chat_repl_status_and_quit_creates_session_ledger() -> None: with runner.isolated_filesystem(): result = runner.invoke(app, ["chat"], input="/status\n/quit\n") @@ -122,6 +421,235 @@ def test_chat_repl_status_and_quit_creates_session_ledger() -> None: assert len(ledgers) == 1 +def test_chat_control_option_cleans_descriptor_after_completed_turn() -> None: + with runner.isolated_filesystem(): + result = runner.invoke( + app, + ["chat", "--control"], + input="plan changes\n/quit\n", + ) + + assert result.exit_code == 0 + assert list(Path(".morphic/control").glob("*.json")) == [] + + +def test_chat_keyboard_interrupt_exits_with_cancelled_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli import chat_command + + def interrupt(coro: object) -> None: + coro.close() # type: ignore[attr-defined] + raise KeyboardInterrupt + + monkeypatch.setattr(chat_command, "_run", interrupt) + + result = runner.invoke(app, ["chat"]) + + assert result.exit_code == 130 + assert "Cancelled." in result.output + + +def test_chat_permission_mode_option_starts_read_only_session() -> None: + with runner.isolated_filesystem(): + result = runner.invoke( + app, + ["chat", "--permission-mode", "read-only"], + input="/status\n/quit\n", + ) + + assert result.exit_code == 0 + assert "mode=read-only" in result.output + ledgers = list(Path(".morphic/sessions").glob("*.jsonl")) + assert len(ledgers) == 1 + events = [ + json.loads(line) + for line in ledgers[0].read_text(encoding="utf-8").splitlines() + ] + assert events[0]["type"] == "session_started" + assert events[0]["payload"]["permission_mode"] == "read-only" + + +def test_chat_read_only_permission_mode_blocks_mutating_tool() -> None: + with runner.isolated_filesystem(): + result = runner.invoke( + app, + ["chat", "--permission-mode", "read-only"], + input='/tools run shell_exec {"cmd":"echo blocked"}\n/quit\n', + ) + + assert result.exit_code == 0 + assert ( + "permission denied: read-only session cannot execute mutating tool: shell_exec" + in result.output + ) + ledgers = list(Path(".morphic/sessions").glob("*.jsonl")) + assert len(ledgers) == 1 + events = [ + json.loads(line) + for line in ledgers[0].read_text(encoding="utf-8").splitlines() + ] + assert any( + event["type"] == "assistant_message" + and "permission denied" in event["payload"]["text"] + for event in events + ) + assert not any(event["type"] == "tool_call_completed" for event in events) + + +def test_chat_repl_hooks_run_records_hook_events_in_current_session() -> None: + with runner.isolated_filesystem(): + hook_dir = Path(".morphic/hooks") + hook_dir.mkdir(parents=True) + (hook_dir / "pre.json").write_text( + json.dumps( + { + "command": "exit 99", + "enabled": True, + "name": "pre-log", + "type": "pre_tool", + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + result = runner.invoke(app, ["chat"], input="/hooks run pre_tool\n/quit\n") + + assert result.exit_code == 0 + assert "hooks type=pre_tool mode=noop succeeded=1 failed=0 skipped=0" in result.output + ledgers = list(Path(".morphic/sessions").glob("*.jsonl")) + assert len(ledgers) == 1 + events = [ + json.loads(line) + for line in ledgers[0].read_text(encoding="utf-8").splitlines() + ] + assert any( + event["type"] == "slash_command" + and event["payload"]["command"] == "/hooks run pre_tool" + for event in events + ) + assert any(event["type"] == "hook_execution_completed" for event in events) + assert any(event["payload"].get("hook_name") == "pre-log" for event in events) + + +def test_chat_repl_hooks_run_respects_shell_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MORPHIC_CHAT_HOOK_EXECUTION", "shell") + with runner.isolated_filesystem(): + hook_dir = Path(".morphic/hooks") + hook_dir.mkdir(parents=True) + (hook_dir / "pre.json").write_text( + json.dumps( + { + "command": "echo repl-hook-ok", + "enabled": True, + "name": "pre-log", + "type": "pre_tool", + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + result = runner.invoke(app, ["chat"], input="/hooks run pre_tool\n/quit\n") + + assert result.exit_code == 0 + assert "hooks type=pre_tool mode=shell succeeded=1 failed=0 skipped=0" in result.output + assert Path(".morphic/audit_log.jsonl").exists() + + +def test_chat_repl_tools_run_records_tool_and_hook_events() -> None: + with runner.isolated_filesystem(): + hook_dir = Path(".morphic/hooks") + hook_dir.mkdir(parents=True) + (hook_dir / "pre.json").write_text( + json.dumps( + { + "command": "exit 99", + "enabled": True, + "name": "pre-log", + "type": "pre_tool", + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + result = runner.invoke( + app, + ["chat"], + input='/tools run fs_read {"path":"README.md"}\n/quit\n', + ) + + assert result.exit_code == 0 + assert "tools tool=fs_read mode=noop success=True" in result.output + ledgers = list(Path(".morphic/sessions").glob("*.jsonl")) + assert len(ledgers) == 1 + events = [ + json.loads(line) + for line in ledgers[0].read_text(encoding="utf-8").splitlines() + ] + assert any( + event["type"] == "slash_command" + and event["payload"]["command"] == '/tools run fs_read {"path":"README.md"}' + for event in events + ) + assert any(event["type"] == "hook_execution_completed" for event in events) + assert any(event["type"] == "tool_call_completed" for event in events) + tool_completed = next( + event for event in events if event["type"] == "tool_call_completed" + ) + assert "not executed" in tool_completed["payload"]["stdout_summary"] + + +def test_chat_repl_tools_run_rejects_invalid_json_args() -> None: + with runner.isolated_filesystem(): + result = runner.invoke(app, ["chat"], input="/tools run fs_read {bad}\n/quit\n") + + assert result.exit_code == 0 + assert "invalid JSON arguments" in result.output + + +def test_chat_repl_tools_run_respects_laee_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MORPHIC_CHAT_TOOL_EXECUTION", "laee") + with runner.isolated_filesystem(): + result = runner.invoke( + app, + ["chat"], + input='/tools run shell_exec {"cmd":"echo tool-ok"}\n/quit\n', + ) + + assert result.exit_code == 0 + assert "tools tool=shell_exec mode=laee success=True" in result.output + audit_log = Path(".morphic/audit_log.jsonl") + assert audit_log.exists() + assert "shell_exec" in audit_log.read_text(encoding="utf-8") + + +def test_chat_repl_tools_run_reports_laee_denied_result( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MORPHIC_CHAT_TOOL_EXECUTION", "laee") + with runner.isolated_filesystem(): + target = Path("delete-me.txt") + target.write_text("keep", encoding="utf-8") + + result = runner.invoke( + app, + ["chat"], + input=f'/tools run fs_delete {{"path":"{target}"}}\n/quit\n', + ) + + assert result.exit_code == 0 + assert "tools tool=fs_delete mode=laee success=False exit_code=1" in result.output + assert "Action requires" in result.output + assert "approval (risk=HIGH, mode=confirm-destructive)" in result.output + assert target.exists() + audit_log = Path(".morphic/audit_log.jsonl") + assert audit_log.exists() + assert "fs_delete" in audit_log.read_text(encoding="utf-8") + + def test_code_one_shot_runs_goal_without_repl() -> None: with runner.isolated_filesystem(): result = runner.invoke(app, ["code", "implement Phase 4"]) @@ -132,6 +660,115 @@ def test_code_one_shot_runs_goal_without_repl() -> None: assert len(ledgers) == 1 +def test_code_benchmark_receipt_emits_canonical_final_line( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli import chat_command + + monkeypatch.setattr(chat_command, "_chat_engine_registry", _FakeEngineRegistry) + monkeypatch.setattr( + chat_command, + "_chat_council_runtime", + lambda **_kwargs: _FakeReceiptCouncilRuntime(), + ) + + with runner.isolated_filesystem(): + result = runner.invoke( + app, + ["code", "--benchmark-receipt", "implement receipt output"], + ) + + assert result.exit_code == 0 + receipt = json.loads(result.output.splitlines()[-1]) + assert receipt == { + "cost_usd": 0.125, + "model": "morphic-control[codex_cli]", + "success": True, + "type": "morphic_benchmark_receipt", + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + + +def test_code_without_benchmark_receipt_preserves_human_output() -> None: + with runner.isolated_filesystem(): + result = runner.invoke(app, ["code", "implement normal output"]) + + assert result.exit_code == 0 + assert "morphic_benchmark_receipt" not in result.output + + +def test_code_benchmark_receipt_does_not_invent_failure_cost( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli import chat_command + + def fail(coro: object) -> None: + coro.close() # type: ignore[attr-defined] + raise RuntimeError("failed locally") + + monkeypatch.setattr(chat_command, "_run", fail) + + result = runner.invoke( + app, + ["code", "--benchmark-receipt", "implement failing output"], + ) + + assert result.exit_code == 1 + assert "morphic_benchmark_receipt" not in result.output + + +def test_code_keyboard_interrupt_exits_with_cancelled_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli import chat_command + + def interrupt(coro: object) -> None: + coro.close() # type: ignore[attr-defined] + raise KeyboardInterrupt + + monkeypatch.setattr(chat_command, "_run", interrupt) + + result = runner.invoke(app, ["code", "fix tests"]) + + assert result.exit_code == 130 + assert "Cancelled." in result.output + + +def test_code_keyboard_interrupt_does_not_invent_benchmark_cost( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli import chat_command + + def interrupt(coro: object) -> None: + coro.close() # type: ignore[attr-defined] + raise KeyboardInterrupt + + monkeypatch.setattr(chat_command, "_run", interrupt) + + result = runner.invoke(app, ["code", "--benchmark-receipt", "fix tests"]) + + assert result.exit_code == 130 + assert "morphic_benchmark_receipt" not in result.output + + +def test_code_permission_mode_option_starts_workspace_write_session() -> None: + with runner.isolated_filesystem(): + result = runner.invoke( + app, + ["code", "--permission-mode", "workspace-write", "implement Phase 4"], + ) + + assert result.exit_code == 0 + ledgers = list(Path(".morphic/sessions").glob("*.jsonl")) + assert len(ledgers) == 1 + events = [ + json.loads(line) + for line in ledgers[0].read_text(encoding="utf-8").splitlines() + ] + assert events[0]["type"] == "session_started" + assert events[0]["payload"]["permission_mode"] == "workspace-write" + + def test_code_route_council_flag_uses_routed_runtime(monkeypatch: pytest.MonkeyPatch) -> None: from interface.cli import chat_command @@ -157,6 +794,94 @@ def fake_council_runtime( assert "Routed response for: implement routed council" in result.output +def test_code_route_direct_flag_uses_single_engine_runtime( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli import chat_command + + calls: list[dict[str, object]] = [] + + def fake_council_runtime(**kwargs: object) -> _FakeCouncilRuntime: + calls.append(kwargs) + return _FakeCouncilRuntime() + + monkeypatch.setattr(chat_command, "_chat_engine_registry", _FakeEngineRegistry) + monkeypatch.setattr(chat_command, "_chat_council_runtime", fake_council_runtime) + + with runner.isolated_filesystem(): + result = runner.invoke( + app, + [ + "code", + "--route-direct", + "--engine", + "codex_cli", + "--permission-mode", + "danger-full-access", + "implement direct routing", + ], + ) + + assert result.exit_code == 0 + assert calls == [ + { + "critic_engine": None, + "direct_engine": "codex_cli", + "leader_engine": None, + "planner_engine": None, + "route_council": False, + "route_direct": True, + } + ] + assert "Routed response for: implement direct routing" in result.output + + +def test_chat_rejects_direct_and_council_modes_together() -> None: + with runner.isolated_filesystem(): + result = runner.invoke( + app, + ["chat", "--route-direct", "--route-council"], + ) + + assert result.exit_code == 2 + assert "mutually exclusive" in result.output + + +def test_code_rejects_invalid_direct_engine() -> None: + with runner.isolated_filesystem(): + result = runner.invoke( + app, + [ + "code", + "--route-direct", + "--engine", + "missing_engine", + "implement direct routing", + ], + ) + + assert result.exit_code == 2 + assert "Invalid direct engine" in result.output + assert "missing_engine" in result.output + + +def test_code_direct_route_requires_supported_explicit_engine() -> None: + with runner.isolated_filesystem(): + result = runner.invoke( + app, + [ + "code", + "--route-direct", + "--permission-mode", + "workspace-write", + "implement direct routing", + ], + ) + + assert result.exit_code == 2 + assert "requires --engine codex_cli or claude_code" in result.output + + def test_code_defaults_to_local_council_mode(monkeypatch: pytest.MonkeyPatch) -> None: from interface.cli import chat_command @@ -214,16 +939,20 @@ def test_code_route_council_role_flags_are_parsed( def fake_council_runtime( *, route_council: bool = False, + route_direct: bool = False, + direct_engine: str | None = None, planner_engine: str | None = None, critic_engine: str | None = None, leader_engine: str | None = None, ) -> _FakeCouncilRuntime: calls.append( - { - "critic_engine": critic_engine, - "leader_engine": leader_engine, - "planner_engine": planner_engine, - "route_council": route_council, + { + "critic_engine": critic_engine, + "direct_engine": direct_engine, + "leader_engine": leader_engine, + "planner_engine": planner_engine, + "route_council": route_council, + "route_direct": route_direct, } ) return _FakeCouncilRuntime() @@ -251,9 +980,11 @@ def fake_council_runtime( assert calls == [ { "critic_engine": "claude_code", + "direct_engine": None, "leader_engine": "gemini_cli", "planner_engine": "codex_cli", "route_council": True, + "route_direct": False, } ] @@ -297,6 +1028,41 @@ def __init__( } +def test_chat_council_runtime_builds_direct_engine_preference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli import chat_command + + class _Container: + route_to_engine = object() + + captured: dict[str, object] = {} + + class _FakeDirectRuntime(_FakeCouncilRuntime): + def __init__( + self, + route_to_engine: object, + *, + preferred_engine: AgentEngineType | None = None, + ) -> None: + captured["preferred_engine"] = preferred_engine + captured["route_to_engine"] = route_to_engine + + monkeypatch.setattr(chat_command, "_get_container", lambda: _Container()) + monkeypatch.setattr(chat_command, "RouteChatDirectRuntime", _FakeDirectRuntime) + + runtime = chat_command._chat_council_runtime( + route_direct=True, + direct_engine="codex_cli", + ) + + assert isinstance(runtime, _FakeDirectRuntime) + assert captured == { + "preferred_engine": AgentEngineType.CODEX_CLI, + "route_to_engine": _Container.route_to_engine, + } + + def test_role_engine_preferences_reject_invalid_engine_id() -> None: with pytest.raises(ValueError) as exc_info: _role_engine_preferences( @@ -340,6 +1106,232 @@ async def test_chat_repl_run_goal_uses_injected_council_runtime(tmp_path) -> Non assert "codex_cli" in ledger.read_text(encoding="utf-8") +@pytest.mark.asyncio +async def test_chat_repl_forwards_streamed_events_to_injected_progress_sink(tmp_path) -> None: + sink = _CollectingEventSink() + + output = await ChatRepl( + workspace_root=tmp_path, + council_runtime=_FakeStreamingCouncilRuntime(), + engine_event_observer=sink, + ).run_goal(goal="fix tests") + + assert output == "Completed: fix tests" + assert [event.type for event in sink.events] == [AgentEngineEventType.RUN_STARTED] + + +@pytest.mark.asyncio +async def test_chat_repl_cancels_only_active_turn_then_continues( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + lines = iter(["fix tests", "/quit"]) + monkeypatch.setattr("builtins.input", lambda _prompt: next(lines)) + runtime = _BlockingStreamingCouncilRuntime() + turn_controller = ActiveTurnController() + repl_task = asyncio.create_task( + ChatRepl( + workspace_root=tmp_path, + council_runtime=runtime, + turn_controller=turn_controller, + ).run() + ) + await runtime.started.wait() + + assert turn_controller.cancel_active_turn() is True + session = await repl_task + + assert session.status.value == "ended" + assert "Turn cancelled." in capsys.readouterr().out + ledger = next((tmp_path / ".morphic" / "sessions").glob("*.jsonl")) + events = [json.loads(line) for line in ledger.read_text().splitlines()] + assert [event["sequence"] for event in events] == list(range(len(events))) + assert [event["type"] for event in events[-5:]] == [ + "user_message", + "engine_event", + "turn_cancelled", + "slash_command", + "session_ended", + ] + + +@pytest.mark.asyncio +async def test_chat_repl_exposes_opt_in_remote_turn_cancellation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + lines = iter(["fix tests", "/quit"]) + monkeypatch.setattr("builtins.input", lambda _prompt: next(lines)) + runtime = _BlockingStreamingCouncilRuntime() + repl_task = asyncio.create_task( + ChatRepl( + workspace_root=tmp_path, + council_runtime=runtime, + control_enabled=True, + ).run() + ) + await runtime.started.wait() + descriptor_path = next((tmp_path / ".morphic" / "control").glob("*.json")) + session_id = json.loads(descriptor_path.read_text())["session_id"] + + response = await send_chat_control_command( + workspace_root=tmp_path, + session_id=session_id, + command="cancel", + ) + session = await repl_task + + assert response["cancelled"] is True + assert session.status.value == "ended" + assert descriptor_path.exists() is False + + +def test_chat_control_status_command_is_registered( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli.commands import chat_control + + def fake_run(coro: object) -> dict[str, object]: + coro.close() # type: ignore[attr-defined] + return { + "active_turn": True, + "ok": True, + "session_id": "chat-control-cli", + } + + monkeypatch.setattr(chat_control, "_run", fake_run) + + result = runner.invoke( + app, + [ + "chat-control", + "status", + "--session", + "chat-control-cli", + "--json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.output)["active_turn"] is True + + +@pytest.mark.asyncio +async def test_chat_repl_steers_replacement_into_resumed_native_session( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + lines = iter(["broad implementation", "/quit"]) + monkeypatch.setattr("builtins.input", lambda _prompt: next(lines)) + runtime = _SteerableStreamingCouncilRuntime() + repl_task = asyncio.create_task( + ChatRepl( + workspace_root=tmp_path, + council_runtime=runtime, + control_enabled=True, + ).run() + ) + await runtime.first_turn_started.wait() + descriptor_path = next((tmp_path / ".morphic" / "control").glob("*.json")) + session_id = json.loads(descriptor_path.read_text())["session_id"] + + response = await send_chat_control_command( + workspace_root=tmp_path, + session_id=session_id, + command="steer", + prompt="/quit but continue native work", + ) + session = await repl_task + + assert response["steered"] is True + assert runtime.messages == ["broad implementation", "/quit but continue native work"] + assert runtime.resumed_native_session_id == "thread-steered" + assert session.status.value == "ended" + ledger = next((tmp_path / ".morphic" / "sessions").glob("*.jsonl")) + events = [json.loads(line) for line in ledger.read_text().splitlines()] + assert [ + event["payload"]["text"] + for event in events + if event["type"] == "user_message" + ] == ["broad implementation", "/quit but continue native work"] + event_types = [event["type"] for event in events] + cancelled_index = event_types.index("turn_cancelled") + assert event_types[cancelled_index : cancelled_index + 3] == [ + "turn_cancelled", + "turn_steered", + "user_message", + ] + steered_event = events[cancelled_index + 1] + assert steered_event["payload"] == { + "replacement_prompt_bytes": len(b"/quit but continue native work"), + "source": "turn_controller", + } + + +def test_chat_control_steer_command_is_registered( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from interface.cli.commands import chat_control + + def fake_run(coro: object) -> dict[str, object]: + coro.close() # type: ignore[attr-defined] + return { + "active_turn": True, + "ok": True, + "session_id": "chat-control-cli", + "steered": True, + } + + monkeypatch.setattr(chat_control, "_run", fake_run) + + result = runner.invoke( + app, + [ + "chat-control", + "steer", + "focus on tests", + "--session", + "chat-control-cli", + "--json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.output)["steered"] is True + + +@pytest.mark.asyncio +async def test_native_event_progress_renderer_is_concise_and_hides_raw_payload() -> None: + lines: list[str] = [] + renderer = NativeEventProgressRenderer(printer=lines.append) + + await renderer.publish( + AgentEngineEvent( + type=AgentEngineEventType.TOOL_STARTED, + engine=AgentEngineType.CODEX_CLI, + sequence=1, + session_id="thread-ui", + item_type="command_execution", + text="pytest tests/unit/ -q", + payload={"secret": "must-not-render", "reasoning": "private"}, + ) + ) + await renderer.publish( + AgentEngineEvent( + type=AgentEngineEventType.ASSISTANT_MESSAGE, + engine=AgentEngineType.CODEX_CLI, + sequence=2, + text="final answer is rendered elsewhere", + payload={}, + ) + ) + + assert lines == ["codex_cli | tool started: pytest tests/unit/ -q"] + assert "must-not-render" not in "".join(lines) + assert "private" not in "".join(lines) + + def test_render_approval_request_includes_risk_and_action() -> None: request = ApprovalRequest( id="approval-1", diff --git a/tests/unit/interface/test_chat_control_transport.py b/tests/unit/interface/test_chat_control_transport.py new file mode 100644 index 0000000..31ce85d --- /dev/null +++ b/tests/unit/interface/test_chat_control_transport.py @@ -0,0 +1,240 @@ +"""Authenticated loopback chat control transport tests.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import stat + +import pytest + +from interface.cli.chat_control_transport import ( + ChatControlServer, + send_chat_control_command, +) +from interface.cli.turn_control import ActiveTurnController, TurnCancelledError + + +@pytest.mark.asyncio +async def test_control_server_reports_and_cancels_active_turn(tmp_path) -> None: + controller = ActiveTurnController() + operation_started = asyncio.Event() + + async def operation() -> None: + operation_started.set() + await asyncio.Event().wait() + + controlled = asyncio.create_task(controller.run(operation)) + await operation_started.wait() + + async with ChatControlServer( + workspace_root=tmp_path, + session_id="chat-control-1", + turn_controller=controller, + ) as server: + descriptor = json.loads(server.descriptor_path.read_text(encoding="utf-8")) + assert descriptor["host"] == "127.0.0.1" + assert descriptor["session_id"] == "chat-control-1" + assert descriptor["token"] + assert stat.S_IMODE(server.descriptor_path.stat().st_mode) == 0o600 + + status = await send_chat_control_command( + workspace_root=tmp_path, + session_id="chat-control-1", + command="status", + ) + assert status == { + "active_turn": True, + "ok": True, + "session_id": "chat-control-1", + } + + cancelled = await send_chat_control_command( + workspace_root=tmp_path, + session_id="chat-control-1", + command="cancel", + ) + assert cancelled == { + "active_turn": True, + "cancelled": True, + "ok": True, + "session_id": "chat-control-1", + } + + with pytest.raises(TurnCancelledError): + await controlled + assert server.descriptor_path.exists() is False + + +@pytest.mark.asyncio +async def test_control_server_rejects_invalid_token_without_cancelling(tmp_path) -> None: + controller = ActiveTurnController() + operation_started = asyncio.Event() + + async def operation() -> None: + operation_started.set() + await asyncio.Event().wait() + + controlled = asyncio.create_task(controller.run(operation)) + await operation_started.wait() + + async with ChatControlServer( + workspace_root=tmp_path, + session_id="chat-control-auth", + turn_controller=controller, + ) as server: + descriptor = json.loads(server.descriptor_path.read_text(encoding="utf-8")) + + async def request(payload: dict[str, object]) -> dict[str, object]: + reader, writer = await asyncio.open_connection( + host=descriptor["host"], + port=descriptor["port"], + ) + writer.write(json.dumps(payload, sort_keys=True).encode("utf-8") + b"\n") + await writer.drain() + response = json.loads(await reader.readline()) + writer.close() + await writer.wait_closed() + return response + + unauthorized = await request( + { + "command": "cancel", + "session_id": "chat-control-auth", + "token": "wrong-token", + } + ) + mismatched = await request( + { + "command": "cancel", + "session_id": "another-session", + "token": descriptor["token"], + } + ) + unsupported = await request( + { + "command": "restart", + "session_id": "chat-control-auth", + "token": descriptor["token"], + } + ) + invalid_steer = await request( + { + "command": "steer", + "session_id": "chat-control-auth", + "token": descriptor["token"], + } + ) + + assert unauthorized == {"error": "unauthorized", "ok": False} + assert mismatched == {"error": "session_mismatch", "ok": False} + assert unsupported == {"error": "unsupported_command", "ok": False} + assert invalid_steer == {"error": "invalid_steer_prompt", "ok": False} + assert controller.has_active_turn is True + + controlled.cancel() + with pytest.raises(asyncio.CancelledError): + await controlled + + +@pytest.mark.asyncio +async def test_control_client_reports_inactive_when_descriptor_is_absent(tmp_path) -> None: + response = await send_chat_control_command( + workspace_root=tmp_path, + session_id="chat-control-idle", + command="status", + ) + + assert response == { + "active_turn": False, + "ok": True, + "session_id": "chat-control-idle", + } + + +@pytest.mark.asyncio +async def test_control_client_rejects_non_loopback_descriptor(tmp_path) -> None: + session_id = "chat-control-host" + digest = hashlib.sha256(session_id.encode("utf-8")).hexdigest()[:32] + descriptor_path = tmp_path / ".morphic" / "control" / f"{digest}.json" + descriptor_path.parent.mkdir(parents=True) + descriptor_path.write_text( + json.dumps( + { + "host": "0.0.0.0", + "port": 8000, + "protocol_version": 1, + "session_id": session_id, + "token": "token", + } + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="loopback"): + await send_chat_control_command( + workspace_root=tmp_path, + session_id=session_id, + command="status", + ) + + +@pytest.mark.asyncio +async def test_control_client_rejects_unsupported_command(tmp_path) -> None: + with pytest.raises(ValueError, match="unsupported"): + await send_chat_control_command( + workspace_root=tmp_path, + session_id="chat-control-command", + command="restart", + ) + + +@pytest.mark.asyncio +async def test_control_server_queues_bounded_steer_and_cancels_turn(tmp_path) -> None: + controller = ActiveTurnController() + operation_started = asyncio.Event() + + async def operation() -> None: + operation_started.set() + await asyncio.Event().wait() + + controlled = asyncio.create_task(controller.run(operation)) + await operation_started.wait() + + async with ChatControlServer( + workspace_root=tmp_path, + session_id="chat-control-steer", + turn_controller=controller, + ): + response = await send_chat_control_command( + workspace_root=tmp_path, + session_id="chat-control-steer", + command="steer", + prompt=" continue with focused tests ", + ) + + assert response == { + "active_turn": True, + "ok": True, + "session_id": "chat-control-steer", + "steered": True, + } + with pytest.raises(TurnCancelledError): + await controlled + assert controller.take_steer_prompt() == "continue with focused tests" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt", [" ", "x" * 2049]) +async def test_control_client_rejects_invalid_steer_prompt_before_cancel( + tmp_path, + prompt: str, +) -> None: + with pytest.raises(ValueError, match="steer prompt"): + await send_chat_control_command( + workspace_root=tmp_path, + session_id="chat-control-invalid-steer", + command="steer", + prompt=prompt, + ) diff --git a/tests/unit/interface/test_hooks_cli.py b/tests/unit/interface/test_hooks_cli.py new file mode 100644 index 0000000..cfc6e0f --- /dev/null +++ b/tests/unit/interface/test_hooks_cli.py @@ -0,0 +1,83 @@ +"""CLI tests for manual Morphic Chat hook execution.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from interface.cli.main import app + +runner = CliRunner() + + +def test_hooks_run_json_executes_hook_with_noop_default() -> None: + with runner.isolated_filesystem(): + hook_dir = Path(".morphic/hooks") + hook_dir.mkdir(parents=True) + (hook_dir / "pre.json").write_text( + json.dumps( + { + "command": "exit 99", + "enabled": True, + "name": "pre-log", + "type": "pre_tool", + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + result = runner.invoke(app, ["hooks", "run", "pre_tool", "--json"]) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["hook_execution_mode"] == "noop" + assert payload["hook_type"] == "pre_tool" + assert payload["summary"] == {"failed": 0, "skipped": 0, "succeeded": 1} + assert payload["results"][0]["hook_name"] == "pre-log" + assert payload["results"][0]["success"] is True + assert "not executed" in payload["results"][0]["stdout_summary"] + ledgers = list(Path(".morphic/sessions").glob("*.jsonl")) + assert len(ledgers) == 1 + assert "hook_execution_completed" in ledgers[0].read_text(encoding="utf-8") + + +def test_hooks_run_rejects_invalid_hook_type() -> None: + with runner.isolated_filesystem(): + result = runner.invoke(app, ["hooks", "run", "before_everything", "--json"]) + + assert result.exit_code == 2 + assert "Invalid hook type" in result.output + + +def test_hooks_run_shell_opt_in_executes_hook_and_writes_audit_log( + monkeypatch, +) -> None: + monkeypatch.setenv("MORPHIC_CHAT_HOOK_EXECUTION", "shell") + with runner.isolated_filesystem(): + hook_dir = Path(".morphic/hooks") + hook_dir.mkdir(parents=True) + (hook_dir / "pre.json").write_text( + json.dumps( + { + "command": "echo hook-ok", + "enabled": True, + "name": "pre-log", + "type": "pre_tool", + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + result = runner.invoke(app, ["hooks", "run", "pre_tool", "--json"]) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["hook_execution_mode"] == "shell" + assert payload["results"][0]["stdout_summary"] == "hook-ok" + audit_log = Path(".morphic/audit_log.jsonl") + assert audit_log.exists() + assert "shell_exec" in audit_log.read_text(encoding="utf-8") diff --git a/tests/unit/interface/test_turn_control.py b/tests/unit/interface/test_turn_control.py new file mode 100644 index 0000000..526aa92 --- /dev/null +++ b/tests/unit/interface/test_turn_control.py @@ -0,0 +1,107 @@ +"""Active chat turn cancellation control tests.""" + +from __future__ import annotations + +import asyncio +import signal +from typing import Any + +import pytest + +from interface.cli.turn_control import ActiveTurnController, TurnCancelledError + + +@pytest.mark.asyncio +async def test_controller_cancels_only_registered_active_turn() -> None: + controller = ActiveTurnController() + started = asyncio.Event() + cleaned_up = asyncio.Event() + + async def operation() -> None: + started.set() + try: + await asyncio.Event().wait() + finally: + cleaned_up.set() + + controlled = asyncio.create_task(controller.run(operation)) + await started.wait() + + assert controller.has_active_turn is True + assert controller.cancel_active_turn() is True + assert controller.cancel_active_turn() is True + with pytest.raises(TurnCancelledError): + await controlled + + assert cleaned_up.is_set() + assert controller.has_active_turn is False + assert controller.cancel_active_turn() is False + + +@pytest.mark.asyncio +async def test_controller_does_not_translate_external_caller_cancellation() -> None: + controller = ActiveTurnController() + started = asyncio.Event() + + async def operation() -> None: + started.set() + await asyncio.Event().wait() + + controlled = asyncio.create_task(controller.run(operation)) + await started.wait() + controlled.cancel() + + with pytest.raises(asyncio.CancelledError): + await controlled + + assert controller.has_active_turn is False + + +@pytest.mark.asyncio +async def test_controller_routes_sigint_to_active_turn_and_restores_handler( + monkeypatch: pytest.MonkeyPatch, +) -> None: + installed_handlers: list[Any] = [] + monkeypatch.setattr(signal, "getsignal", lambda _signal: signal.SIG_DFL) + monkeypatch.setattr( + signal, + "signal", + lambda _signal, handler: installed_handlers.append(handler), + ) + controller = ActiveTurnController() + started = asyncio.Event() + + async def operation() -> None: + started.set() + await asyncio.Event().wait() + + controlled = asyncio.create_task(controller.run(operation)) + await started.wait() + active_handler = installed_handlers[0] + + active_handler(signal.SIGINT, None) + + with pytest.raises(TurnCancelledError): + await controlled + assert installed_handlers[-1] == signal.SIG_DFL + + +@pytest.mark.asyncio +async def test_controller_queues_only_first_steer_prompt() -> None: + controller = ActiveTurnController() + started = asyncio.Event() + + async def operation() -> None: + started.set() + await asyncio.Event().wait() + + controlled = asyncio.create_task(controller.run(operation)) + await started.wait() + + assert controller.steer_active_turn("replacement prompt") is True + assert controller.steer_active_turn("must not replace") is False + with pytest.raises(TurnCancelledError): + await controlled + + assert controller.take_steer_prompt() == "replacement prompt" + assert controller.take_steer_prompt() is None diff --git a/uv.lock b/uv.lock index e96dc69..4803ca7 100644 --- a/uv.lock +++ b/uv.lock @@ -2358,6 +2358,7 @@ dependencies = [ { name = "apscheduler" }, { name = "asyncpg" }, { name = "celery", extra = ["redis"] }, + { name = "cryptography" }, { name = "ddgs" }, { name = "fastapi" }, { name = "httpx" }, @@ -2401,6 +2402,7 @@ requires-dist = [ { name = "apscheduler", specifier = ">=3.10" }, { name = "asyncpg", specifier = ">=0.30" }, { name = "celery", extras = ["redis"], specifier = ">=5.4" }, + { name = "cryptography", specifier = ">=46.0.5" }, { name = "ddgs", specifier = ">=9.11.4" }, { name = "fastapi", specifier = ">=0.115" }, { name = "google-adk", marker = "extra == 'adk'", specifier = ">=1.0" },