From c7850e49d8cc07169c46eef98cce6dfa509039bd Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 14:21:16 -0400 Subject: [PATCH 01/22] test(emit): scope the artifacts-dir litter check to files The content-addressed object store lands a declared objects/ subtree under .loop/artifacts/. Listing files only keeps the anti-litter teeth (a duplicate bundle or an orphan .staged still fails) without pinning the directory shape. Green against the unmodified writer as well as the new one. --- scripts/test_verify_evidence_writer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/test_verify_evidence_writer.py b/scripts/test_verify_evidence_writer.py index 3d946b0..4bb5213 100644 --- a/scripts/test_verify_evidence_writer.py +++ b/scripts/test_verify_evidence_writer.py @@ -146,7 +146,10 @@ def test_rewrite_of_the_same_iteration_is_idempotent_in_path(tmp_path): first, second = _write(workspace), _write(workspace) assert first["bundle"] == second["bundle"] and first["evidence"] == second["evidence"] assert first["sha256"] == second["sha256"] - assert [p.name for p in sorted((workspace / ".loop" / "artifacts").iterdir())] == ["verify-iter5.json"] + # Files only: the content-addressed object store is a declared subtree under + # artifacts/, not litter. A duplicate bundle or an orphan .staged still fails. + assert [p.name for p in sorted((workspace / ".loop" / "artifacts").iterdir()) + if p.is_file()] == ["verify-iter5.json"] def test_writer_refuses_a_workspace_with_no_contract(tmp_path): From d0f4fc0b22dd285a0a2f0209adfa1c26264cf3c9 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 14:21:17 -0400 Subject: [PATCH 02/22] feat(emit): pure evidence builder, content-addressed object store, bound digest set --- loop/emit.py | 128 ++++++++++++++++---- scripts/test_verify_evidence_objects.py | 150 ++++++++++++++++++++++++ 2 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 scripts/test_verify_evidence_objects.py diff --git a/loop/emit.py b/loop/emit.py index c85b839..660772a 100644 --- a/loop/emit.py +++ b/loop/emit.py @@ -11,6 +11,7 @@ import json import os import tempfile +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping, Sequence @@ -29,7 +30,7 @@ _validate_terminal, _validation_mode, ) -from .evidence import EVIDENCE_SCHEMA_ID +from .evidence import EVIDENCE_SCHEMA_ID, artifact_object_path, validate_evidence from .paths import ARTIFACTS_DIR_NAME, EVIDENCE_DIR_NAME, resolve_loop_paths from .scaffold import scaffold from .verifier import criterion_partition, verification_policy_digest @@ -407,23 +408,36 @@ def sync_state_to_projection(target: str | Path, projection: dict[str, Any]) -> DEFAULT_VERIFIER_IDENTITY = "loop.run" -def write_verify_evidence( +@dataclass(frozen=True) +class BuiltEvidence: + """Every byte and digest of one verify write, computed without touching disk. + + Separating the computation from the write lets a caller commit to the exact + artifact hashes BEFORE anything is placed, so the digests a durable event + carries and the bytes on disk cannot disagree. + """ + + bundle_path: Path + bundle_text: str + sha256: str + record_path: Path + record_text: str + record_sha256: str + object_path: Path + artifact_hashes: tuple[dict[str, str], ...] + + +def build_verify_evidence( target: str | Path, *, run_id: str, iteration_id: int, task: Mapping[str, Any], passed: bool, code_identity: Mapping[str, Any], summary: str = "", executor: str | None = None, verifier_identity: str | None = None, attempt: int | None = None, -) -> dict[str, Any]: - """Write one verify bundle and its hashed evidence@1 record. +) -> BuiltEvidence: + """Render one verify bundle, its evidence@1 record, and their bound digests. - ``code_identity`` is REQUIRED and is never derived here: only the caller knows - which verifier actually ran (see loop.verifier.executed_verifier_identity vs - injected_verifier_identity). Deriving it from ``task['verify']`` would record a - command that an injected verifier never executed. - - The bundle is the artifact (metrics-compatible; carries verifier identity, the - verdict's source, and the DECLARED criterion partition); the record is the - schema-validated pointer that commits to the bundle's bytes. Identities are - recorded, never inferred: an unsupplied executor is the literal ``unattributed`` - and an unsupplied attempt is ``null``. + Pure with respect to the workspace: it reads the contract and writes nothing. + The record it renders is byte-identical to what ``write_verify_evidence`` + places, and is schema-validated here so an invalid record is refused before + any file exists rather than after. """ paths = _require_contract(target) iteration_id = _require_iteration_id(iteration_id) @@ -450,12 +464,8 @@ def write_verify_evidence( "partition": criterion_partition(task), } bundle_path = paths.loop_dir / ARTIFACTS_DIR_NAME / f"verify-iter{iteration_id}.json" - bundle_path.parent.mkdir(parents=True, exist_ok=True) bundle_text = json.dumps(bundle, indent=2, sort_keys=True) + "\n" - # Staged under a name metrics does not rglob, so a failure before the record is - # written cannot leave an orphan green bundle for FCR to count. - staged = bundle_path.with_name(bundle_path.name + ".staged") - _atomic_write_text(staged, bundle_text) + bundle_sha256 = hashlib.sha256(bundle_text.encode("utf-8")).hexdigest() now = datetime.now(timezone.utc).isoformat(timespec="seconds") record = { @@ -463,7 +473,7 @@ def write_verify_evidence( "id": f"{run_id}:{iteration_id}:verify", "kind": "verify-bundle", "uri": bundle_path.relative_to(paths.workspace).as_posix(), - "sha256": hashlib.sha256(bundle_text.encode("utf-8")).hexdigest(), + "sha256": bundle_sha256, "media_type": "application/json", "created_at": now, "produced_by": {"run_id": run_id, "task_id": task.get("id"), "attempt": attempt, @@ -473,12 +483,82 @@ def write_verify_evidence( "code_digest_basis": identity["code_digest_basis"], "policy_digest": identity["policy_digest"]}, } + validation = validate_evidence(record, mode="basic") + if not validation["ok"]: + raise EmitError(f"evidence record failed schema validation: {validation['issues']}") + record_path = paths.loop_dir / EVIDENCE_DIR_NAME / f"evidence-iter{iteration_id}.json" - record_path.parent.mkdir(parents=True, exist_ok=True) + record_text = json.dumps(record, indent=2, sort_keys=True) + "\n" + record_sha256 = hashlib.sha256(record_text.encode("utf-8")).hexdigest() + object_path = artifact_object_path(paths.workspace, bundle_sha256) + artifact_hashes = tuple( + {"path": path.relative_to(paths.workspace).as_posix(), "sha256": digest} + for path, digest in ((bundle_path, bundle_sha256), (record_path, record_sha256), + (object_path, bundle_sha256)) + ) + return BuiltEvidence( + bundle_path=bundle_path, bundle_text=bundle_text, sha256=bundle_sha256, + record_path=record_path, record_text=record_text, record_sha256=record_sha256, + object_path=object_path, artifact_hashes=artifact_hashes, + ) + + +def write_verify_evidence( + target: str | Path, *, run_id: str, iteration_id: int, task: Mapping[str, Any], passed: bool, + code_identity: Mapping[str, Any], summary: str = "", executor: str | None = None, + verifier_identity: str | None = None, attempt: int | None = None, + built: BuiltEvidence | None = None, +) -> dict[str, Any]: + """Write one verify bundle, its content-addressed object, and its evidence@1 record. + + ``code_identity`` is REQUIRED and is never derived here: only the caller knows + which verifier actually ran (see loop.verifier.executed_verifier_identity vs + injected_verifier_identity). Deriving it from ``task['verify']`` would record a + command that an injected verifier never executed. + + The bundle is the artifact (metrics-compatible; carries verifier identity, the + verdict's source, and the DECLARED criterion partition); the record is the + schema-validated pointer that commits to the bundle's bytes. Identities are + recorded, never inferred: an unsupplied executor is the literal ``unattributed`` + and an unsupplied attempt is ``null``. + + ``built`` lets a caller that already rendered the write place exactly those + bytes, so the digests it committed to elsewhere describe what landed. + """ + if built is None: + built = build_verify_evidence( + target, run_id=run_id, iteration_id=iteration_id, task=task, passed=passed, + code_identity=code_identity, summary=summary, executor=executor, + verifier_identity=verifier_identity, attempt=attempt, + ) + + # The object is placed FIRST and never replaced: it is the recovery copy the + # evidence record's digest still resolves to after the mutable bundle path is + # overwritten. + try: + built.object_path.parent.mkdir(parents=True, exist_ok=True) + _atomic_create_text(built.object_path, built.bundle_text) + except FileExistsError: + existing = hashlib.sha256(built.object_path.read_bytes()).hexdigest() + if existing != built.sha256: + raise EmitError( + f"object store holds different bytes for {built.sha256}: refusing to overwrite " + f"a content-addressed object (found {existing})") + except OSError as exc: + raise EmitError(f"object store write failed at {built.object_path}: {exc}") from exc + + built.bundle_path.parent.mkdir(parents=True, exist_ok=True) + # Staged under a name metrics does not rglob, so a failure before the record is + # written cannot leave an orphan green bundle for FCR to count. + staged = built.bundle_path.with_name(built.bundle_path.name + ".staged") + _atomic_write_text(staged, built.bundle_text) + + built.record_path.parent.mkdir(parents=True, exist_ok=True) try: - _atomic_write_text(record_path, json.dumps(record, indent=2, sort_keys=True) + "\n") + _atomic_write_text(built.record_path, built.record_text) except BaseException: staged.unlink(missing_ok=True) raise - os.replace(staged, bundle_path) - return {"bundle": bundle_path, "evidence": record_path, "sha256": record["sha256"]} + os.replace(staged, built.bundle_path) + return {"bundle": built.bundle_path, "evidence": built.record_path, + "sha256": built.sha256, "object": built.object_path} diff --git a/scripts/test_verify_evidence_objects.py b/scripts/test_verify_evidence_objects.py new file mode 100644 index 0000000..24fa4e3 --- /dev/null +++ b/scripts/test_verify_evidence_objects.py @@ -0,0 +1,150 @@ +"""build_verify_evidence purity, the content-addressed object, and the bound digest set.""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop import emit +from loop.evidence import artifact_object_path +from loop.verifier import executed_verifier_identity + + +def _task(**overrides): + base = {"id": "T-1", "title": "t", "status": "pending", "criterion_ref": "C-1", + "verify": "./scripts/verify-fast.sh", "depends_on": [], "attempts": 0, "evidence": None} + base.update(overrides) + return base + + +def _ws(tmp_path): + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + return workspace + + +def _tree(workspace): + return {str(p.relative_to(workspace)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in workspace.rglob("*") if p.is_file()} + + +def _build(workspace, *, iteration_id=1, task=None, passed=True, **kwargs): + task = task or _task() + kwargs.setdefault("code_identity", executed_verifier_identity(task["verify"], workspace)) + return emit.build_verify_evidence(workspace, run_id="run-1", iteration_id=iteration_id, + task=task, passed=passed, **kwargs) + + +def test_build_writes_nothing(tmp_path): + workspace = _ws(tmp_path) + before = _tree(workspace) + _build(workspace) + assert _tree(workspace) == before + + +def test_build_and_write_agree_on_every_byte(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + written = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), + built=built) + assert written["bundle"].read_text(encoding="utf-8") == built.bundle_text + assert written["evidence"].read_text(encoding="utf-8") == built.record_text + + +def test_write_places_the_content_addressed_object(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + written = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + assert written["object"].is_file() + assert hashlib.sha256(written["object"].read_bytes()).hexdigest() == built.sha256 + + +def test_object_path_matches_artifact_object_path(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + assert built.object_path == artifact_object_path(workspace, built.sha256) + + +def test_artifact_hashes_cover_bundle_object_and_record(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + by_path = {entry["path"]: entry["sha256"] for entry in built.artifact_hashes} + assert len(built.artifact_hashes) == 3 + assert by_path[".loop/artifacts/verify-iter1.json"] == built.sha256 + assert by_path[".loop/evidence/evidence-iter1.json"] == built.record_sha256 + assert by_path[built.object_path.relative_to(workspace).as_posix()] == built.sha256 + + +def test_artifact_hashes_entries_are_workspace_relative_posix(tmp_path): + built = _build(_ws(tmp_path)) + for entry in built.artifact_hashes: + assert not entry["path"].startswith("/") and "\\" not in entry["path"] + assert len(entry["sha256"]) == 64 + + +def test_record_sha256_still_commits_to_the_bundle_bytes(tmp_path): + built = _build(_ws(tmp_path)) + record = json.loads(built.record_text) + assert record["sha256"] == hashlib.sha256(built.bundle_text.encode("utf-8")).hexdigest() + + +def test_rewriting_the_bundle_leaves_the_object_intact(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + written = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + written["bundle"].write_text('{"outcome": "PASS", "passed": true}', encoding="utf-8") + assert hashlib.sha256(written["object"].read_bytes()).hexdigest() == built.sha256 + + +def test_a_second_identical_build_reuses_the_object_without_error(tmp_path): + workspace = _ws(tmp_path) + identity = executed_verifier_identity(_task()["verify"], workspace) + first = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, code_identity=identity) + second = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, code_identity=identity) + assert first["object"] == second["object"] and first["sha256"] == second["sha256"] + + +def test_an_object_collision_with_different_bytes_raises_emit_error(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + built.object_path.parent.mkdir(parents=True, exist_ok=True) + built.object_path.write_text("not the bundle", encoding="utf-8") + with pytest.raises(emit.EmitError, match="object store"): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_task(), + passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + + +def test_a_non_canonicalizable_task_raises_a_typed_emit_error_at_build_time(tmp_path): + workspace = _ws(tmp_path) + with pytest.raises(emit.EmitError, match="canonical"): + _build(workspace, task=_task(criterion_ref=float("nan"))) + + +def test_a_failed_record_write_leaves_no_metrics_visible_bundle_and_no_partial_object(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + (workspace / ".loop" / "evidence").mkdir(parents=True, exist_ok=True) + (workspace / ".loop" / "evidence" / "evidence-iter1.json").mkdir() # write must fail + with pytest.raises((OSError, emit.EmitError)): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_task(), + passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + assert not list((workspace / ".loop" / "artifacts").glob("verify-*.json")) From 2db6118bf6a4ab2888b0f3761a6f7b1258a41ba6 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 14:32:46 -0400 Subject: [PATCH 03/22] feat(runner): bind evidence digests into the iteration event before writing them --- loop/runner.py | 21 +++- scripts/test_evidence_binding_runner.py | 139 ++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 scripts/test_evidence_binding_runner.py diff --git a/loop/runner.py b/loop/runner.py index 25557f5..9d1c29c 100644 --- a/loop/runner.py +++ b/loop/runner.py @@ -252,17 +252,30 @@ def dispatch_once( "task_id": task["id"], "summary": outcome.summary, } + # Rendered BEFORE the append and written AFTER it. Building first is what lets + # the event carry the digests of the very bytes that will land; the builder + # writes nothing, so a SIGKILL at the pre-commit COMMIT still leaves the tree + # byte-identical (test_crash_injection_before_iteration_event_commit_...). + try: + built = emit.build_verify_evidence( + target, run_id=run_id, iteration_id=iteration_id, task=task, + passed=outcome.passed, summary=outcome.summary, code_identity=code_identity, + executor=executor, verifier_identity=verifier_identity, attempt=attempt, + ) + except emit.EmitError as exc: + raise RunnerError( + f"cannot build the verify evidence for iteration {iteration_id}: {exc}" + ) from exc store = SQLiteEventStore(paths.loop_dir / "events.db") _store_append(store, run_id, "iteration_appended", payload, actor="loop.run", + artifact_hashes=list(built.artifact_hashes), expected_sequence=projection["last_sequence"] + 1) - # Evidence is written AFTER the durable append: writing it first would leave a - # bundle behind a SIGKILL at the pre-commit COMMIT and break the zero-write - # crash pin (test_crash_injection_before_iteration_event_commit_...). try: written = emit.write_verify_evidence( target, run_id=run_id, iteration_id=iteration_id, task=task, passed=outcome.passed, summary=outcome.summary, code_identity=code_identity, executor=executor, verifier_identity=verifier_identity, attempt=attempt, + built=built, ) except (OSError, emit.EmitError) as exc: raise RunnerError( @@ -273,4 +286,4 @@ def dispatch_once( task_id=payload["task_id"], notes=payload["summary"]) return {"ok": True, "action": "dispatched", "task_id": task["id"], "outcome": payload["outcome"], "iteration_id": iteration_id, "run_id": run_id, - "evidence": str(written["evidence"])} + "evidence": str(written["evidence"]), "object": str(written["object"])} diff --git a/scripts/test_evidence_binding_runner.py b/scripts/test_evidence_binding_runner.py new file mode 100644 index 0000000..34285e9 --- /dev/null +++ b/scripts/test_evidence_binding_runner.py @@ -0,0 +1,139 @@ +"""dispatch_once binds its evidence digests into the iteration event before writing them.""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop import emit +from loop.chain import compute_event_hash +from loop.events import SQLiteEventStore +from loop.runner import RunnerError, VerifyOutcome, dispatch_once + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + + +def _task(i, deps=(), status="pending"): + return {"id": i, "title": i, "status": status, "criterion_ref": i, "verify": _VERIFY, + "depends_on": list(deps), "attempts": 0, "evidence": None} + + +def _ready(tmp_path, tasks): + """A contract projected to execute-task with the iteration counter still at 0. + + The ramp records the FSM walk to execute-task, not work: intake is reachable + only by iteration_appended, so the ramp events exist, but they carry + iteration_id 0 and no task verdict. The first dispatch is therefore iteration + 1 and every artifact path asserted below is literal. + """ + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": tasks}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + return workspace + + +@pytest.fixture +def ready_workspace(tmp_path): + return _ready(tmp_path, [_task("T-1")]) + + +@pytest.fixture +def two_task_workspace(tmp_path): + return _ready(tmp_path, [_task("T-1"), _task("T-2", ("T-1",))]) + + +def _events(workspace, run_id="run-1"): + return SQLiteEventStore(workspace / ".loop" / "events.db").read(run_id) + + +def _iteration_events(workspace): + """Iterations carrying a task verdict — exactly the ones a dispatch appends.""" + return [e for e in _events(workspace) if e["type"] == "iteration_appended" + and e["payload"]["outcome"] in ("task_passed", "task_failed")] + + +def test_dispatch_binds_the_evidence_digests_into_the_iteration_event(ready_workspace): + dispatch_once(ready_workspace) + hashes = _iteration_events(ready_workspace)[-1]["artifact_hashes"] + paths = {entry["path"] for entry in hashes} + assert len(hashes) == 3 + assert ".loop/artifacts/verify-iter1.json" in paths + assert ".loop/evidence/evidence-iter1.json" in paths + + +def test_bound_digests_match_the_files_on_disk(ready_workspace): + dispatch_once(ready_workspace) + for entry in _iteration_events(ready_workspace)[-1]["artifact_hashes"]: + blob = (ready_workspace / entry["path"]).read_bytes() + assert hashlib.sha256(blob).hexdigest() == entry["sha256"] + + +def test_the_bound_event_hash_covers_the_artifact_hashes(ready_workspace): + dispatch_once(ready_workspace) + event = _iteration_events(ready_workspace)[-1] + assert compute_event_hash(event) == event["event_hash"] + tampered = {**event, "artifact_hashes": []} + assert compute_event_hash(tampered) != event["event_hash"] + + +def test_a_build_failure_commits_no_event(ready_workspace): + tasks_path = ready_workspace / "TASKS.json" + data = json.loads(tasks_path.read_text(encoding="utf-8")) + data["tasks"][0]["criterion_ref"] = float("nan") + tasks_path.write_text(json.dumps(data), encoding="utf-8") + before = len(_events(ready_workspace)) + with pytest.raises(RunnerError): + dispatch_once(ready_workspace) + assert len(_events(ready_workspace)) == before + + +def test_an_evidence_write_failure_after_the_commit_is_still_loud(ready_workspace, monkeypatch): + def boom(*args, **kwargs): + raise OSError("disk full") + monkeypatch.setattr(emit, "write_verify_evidence", boom) + with pytest.raises(RunnerError, match="committed to the event log"): + dispatch_once(ready_workspace) + assert len(_iteration_events(ready_workspace)) == 1 + + +def test_dispatch_result_names_the_object_and_the_record(ready_workspace): + result = dispatch_once(ready_workspace) + assert result["evidence"].endswith("evidence-iter1.json") + assert "objects" in result["object"] + + +def test_injected_verifier_dispatch_binds_its_own_bundle(ready_workspace): + dispatch_once(ready_workspace, verifier=lambda task, ws: VerifyOutcome(True, "injected")) + bundle = json.loads((ready_workspace / ".loop" / "artifacts" / "verify-iter1.json") + .read_text(encoding="utf-8")) + entry = next(e for e in _iteration_events(ready_workspace)[-1]["artifact_hashes"] + if e["path"].endswith("verify-iter1.json")) + assert bundle["verifier"]["source"] == "injected_callable" + assert entry["sha256"] == hashlib.sha256( + (ready_workspace / entry["path"]).read_bytes()).hexdigest() + + +def test_two_dispatches_bind_distinct_artifact_sets(two_task_workspace): + dispatch_once(two_task_workspace) + dispatch_once(two_task_workspace) + first, second = _iteration_events(two_task_workspace)[:2] + assert {e["path"] for e in first["artifact_hashes"]} != {e["path"] for e in second["artifact_hashes"]} + + +def test_terminal_written_carries_no_artifact_hashes(ready_workspace): + dispatch_once(ready_workspace) + dispatch_once(ready_workspace) + terminal = [e for e in _events(ready_workspace) if e["type"] == "terminal_written"] + assert terminal and terminal[-1]["artifact_hashes"] == [] From 20d15b4a378e444b66424a8295620e65a0a9135a Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 14:43:45 -0400 Subject: [PATCH 04/22] feat(doctor): verify the evidence digests bound into the event chain --- loop/runtime.py | 49 ++++++ scripts/test_doctor_evidence_binding.py | 189 ++++++++++++++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 scripts/test_doctor_evidence_binding.py diff --git a/loop/runtime.py b/loop/runtime.py index c41bf78..8e300ab 100644 --- a/loop/runtime.py +++ b/loop/runtime.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import shutil import sqlite3 @@ -309,6 +310,7 @@ def event_consistency_issues( "an anchored chain head was supplied but the event store cannot be read")) return {"present": True, "readable": False, "error_code": exc.code}, unreadable_issues issues = list(status["divergence"]) + list(replay["findings"]) + issues.extend(_bound_evidence_issues(target, mode)) if declares_chain_without_columns: issues.append(ContractIssue( "chain_columns_missing", @@ -330,3 +332,50 @@ def event_consistency_issues( "legal_sequence": replay["legal_sequence"], "chain": {"head": status["chain_head"], "unchained_prefix": status["unchained_prefix"]}, }, issues + + +def _bound_evidence_issues(target: str | Path, mode: str | None) -> list[dict[str, Any]]: + """Re-hash every artifact an event bound into the chain. + + Driven by what each event DECLARES: a legacy event carries an empty + artifact_hashes list and is silent by construction, because the append-only + triggers make a retroactive binding impossible (repo-os-contract.md #22). + """ + _, _run_id, events, _validation = _events(target, mode) + workspace = resolve_loop_paths(target).workspace + issues: list[dict[str, Any]] = [] + for event in events: + for entry in event.get("artifact_hashes") or []: + path = workspace / entry["path"] + try: + blob = path.read_bytes() + except OSError: + issues.append(ContractIssue( + "missing_bound_evidence", + f"event {event['event_id']} (sequence {event['sequence']}) bound " + f"{entry['path']} into the chain but it is absent or unreadable")) + continue + actual = hashlib.sha256(blob).hexdigest() + if actual != entry["sha256"]: + issues.append(ContractIssue( + "evidence_chain_mismatch", + f"{entry['path']} does not match the digest bound at sequence " + f"{event['sequence']}: expected {entry['sha256']}, found {actual} — " + f"the original bytes may remain at " + f".loop/artifacts/objects/{entry['sha256'][:2]}/{entry['sha256']}")) + return issues + + +def bound_artifact_digests(target: str | Path, mode: str | None = None) -> dict[str, str] | None: + """{workspace-relative POSIX path: sha256} for every artifact any event bound. + + None means there is no event store, and the caller MUST degrade explicitly and say + so (decision 14) rather than treat absence as satisfaction. An empty dict means a + store exists and bound nothing. An unreadable store raises RuntimeStoreError — an + errored check fails, it never skips (R007). + """ + if not (resolve_loop_paths(target).loop_dir / "events.db").is_file(): + return None + _, _run_id, events, _validation = _events(target, mode) + return {entry["path"]: entry["sha256"] + for event in events for entry in event.get("artifact_hashes") or []} diff --git a/scripts/test_doctor_evidence_binding.py b/scripts/test_doctor_evidence_binding.py new file mode 100644 index 0000000..ffe03d9 --- /dev/null +++ b/scripts/test_doctor_evidence_binding.py @@ -0,0 +1,189 @@ +"""Doctor's chain-binding walk: bound artifacts must still be the bytes that were bound.""" +from __future__ import annotations + +import json + +import pytest + +from loop import emit +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.runner import dispatch_once +from loop.scaffold import scaffold + +try: # the repo's fallback-mode convention + import jsonschema # noqa: F401 + _HAS_JSONSCHEMA = True +except ImportError: # pragma: no cover - exercised in the fallback leg + _HAS_JSONSCHEMA = False + +_MODES = [pytest.param("basic"), + pytest.param("strict", marks=pytest.mark.skipif(not _HAS_JSONSCHEMA, + reason="jsonschema not installed"))] + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RUN_ID = "run-1" + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _tree(workspace): + return sorted(str(p.relative_to(workspace)) for p in workspace.rglob("*") if p.is_file()) + + +def _store_path(workspace): + return workspace / ".loop" / "events.db" + + +def _ready(tmp_path): + """A contract projected to execute-task with the iteration counter still at 0. + + The ramp records the FSM walk to execute-task, not work: intake is reachable + only by iteration_appended, so the ramp events exist, but they carry + iteration_id 0 and no task verdict. The first dispatch is therefore iteration + 1 and every artifact path asserted below is literal. + """ + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [{"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "T-1", + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None}], + }), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(_store_path(workspace)) + store.append(_RUN_ID, "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append(_RUN_ID, "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + # The ramp is written straight to the store, so state.json still says intake. + # Doctor reconciles the two, so the fixture must land where the ramp landed. + state_path = workspace / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["state"] = _RAMP[-1] + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + return workspace + + +def _rebuild_unbound(workspace): + """Replay the stream into a fresh store with nothing bound. + + The append-only triggers make retroactive unbinding impossible, so the only + way a pre-binding contract can exist is the way it was originally written: + every event re-appended verbatim except its artifact_hashes. + """ + events = SQLiteEventStore(_store_path(workspace)).read(_RUN_ID) + for suffix in ("", "-wal", "-shm"): + _store_path(workspace).with_name("events.db" + suffix).unlink(missing_ok=True) + store = SQLiteEventStore(_store_path(workspace)) + for event in events: + store.append(_RUN_ID, event["type"], event["payload"], actor=event["actor"], + event_id=event["event_id"], causation_id=event["causation_id"], + correlation_id=event["correlation_id"], ts=event["ts"], + artifact_hashes=None) + + +@pytest.fixture +def scaffolded_workspace(tmp_path): + target = tmp_path / "scaffolded" + scaffold(target) + return target + + +@pytest.fixture +def ready_workspace(tmp_path): + return _ready(tmp_path) + + +@pytest.fixture +def legacy_workspace(tmp_path): + workspace = _ready(tmp_path) + dispatch_once(workspace) + _rebuild_unbound(workspace) + return workspace + + +@pytest.fixture +def corrupt_store_workspace(tmp_path): + target = tmp_path / "corrupt" + scaffold(target) + _store_path(target).write_text("not sqlite", encoding="utf-8") + return target + + +def test_absent_store_adds_no_binding_issue(scaffolded_workspace): + assert doctor_report(scaffolded_workspace)["event_store"] == {"present": False} + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_clean_runner_dispatch_is_doctor_clean(ready_workspace, mode): + dispatch_once(ready_workspace) + report = doctor_report(ready_workspace, mode=mode) + assert report["ok"] is True and report["issues"] == [] + + +def test_rewriting_a_bound_bundle_is_reported(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").write_text( + json.dumps({"outcome": "PASS", "passed": True}), encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(ready_workspace)) + + +def test_rewriting_a_bound_record_is_reported(ready_workspace): + dispatch_once(ready_workspace) + path = ready_workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(path.read_text(encoding="utf-8")) + record["verified_by"]["by"] = "somebody-else" + path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(ready_workspace)) + + +def test_deleting_a_bound_pair_is_reported(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").unlink() + (ready_workspace / ".loop" / "evidence" / "evidence-iter1.json").unlink() + assert "missing_bound_evidence" in _codes(doctor_report(ready_workspace)) + + +def test_the_object_survives_and_is_named_as_the_recovery_source(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").write_text("{}", encoding="utf-8") + messages = " ".join(issue["message"] for issue in doctor_report(ready_workspace)["issues"]) + assert "objects/" in messages + assert any(p.startswith(".loop/artifacts/objects/") + for p in _tree(ready_workspace)) + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_legacy_event_without_artifact_hashes_is_not_a_finding(legacy_workspace, mode): + report = doctor_report(legacy_workspace, mode=mode) + assert "missing_bound_evidence" not in _codes(report) + assert "evidence_chain_mismatch" not in _codes(report) + + +def test_binding_check_writes_nothing(ready_workspace): + dispatch_once(ready_workspace) + before = _tree(ready_workspace) + doctor_report(ready_workspace) + assert _tree(ready_workspace) == before + + +def test_binding_issues_never_add_a_doctor_key(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").unlink() + report = doctor_report(ready_workspace) + assert set(report["event_store"]) == { + "present", "readable", "run_id", "event_count", "state_json_agrees", + "deterministic", "legal_sequence", "chain"} + + +def test_a_store_that_cannot_be_read_still_reports_its_own_error_code(corrupt_store_workspace): + report = doctor_report(corrupt_store_workspace) + assert report["event_store"]["readable"] is False + assert "missing_bound_evidence" not in _codes(report) From 51a798e5820f708bcbadc4b81fc0555e2b4f78f7 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 15:13:13 -0400 Subject: [PATCH 05/22] feat(doctor): hash-verify discovered evidence and compare the recorded goalpost --- loop/contract.py | 71 ++++++++++- scripts/test_doctor_evidence.py | 20 ++- scripts/test_doctor_evidence_verification.py | 124 +++++++++++++++++++ 3 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 scripts/test_doctor_evidence_verification.py diff --git a/loop/contract.py b/loop/contract.py index 9ec0e44..7927cb0 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -6,6 +6,7 @@ from typing import Any, Mapping from . import fsm +from .chain import ChainHashError from .completion import ( CompletionPolicyError, criteria_satisfy_completion, @@ -635,6 +636,7 @@ def _validate_jsonl(path: Path, schema_key: str, mode: str, issues: list[dict]) _RUNNER_BUNDLE_RE = re.compile(r"verify-iter([0-9]+)\.json") +_EVIDENCE_RECORD_RE = re.compile(r"evidence-iter([0-9]+)\.json") def _self_verified(record: Mapping[str, Any]) -> bool: @@ -681,33 +683,96 @@ def _orphan_bundle_issues(paths: LoopPaths, issues: list[dict]) -> None: bundle_path)) +def _policy_digest_issues( + paths: LoopPaths, + records: list[tuple[int | None, Path, dict]], + issues: list[dict], +) -> None: + """Compare the LATEST record per task against the live TASKS.json goalpost. + + Only the latest record backs the task's current claim; older records describe + goalposts that were current when they were written, and comparing them would + make every honest re-verification a permanent failure (repo-os-contract.md §17). + A record naming a task that is no longer in TASKS.json is not compared at all — + a renamed or removed task is a replan, not a forgery. + """ + from .verifier import verification_policy_digest + + tasks = _read_json(paths.tasks, []) # already reported by the contract read + declared = tasks.get("tasks") if isinstance(tasks, dict) else None + entries = {t["id"]: t for t in (declared if isinstance(declared, list) else []) + if isinstance(t, dict) and isinstance(t.get("id"), str)} + latest: dict[str, tuple[int, Path, dict]] = {} + for iteration_id, record_path, data in records: + if not isinstance(iteration_id, int): + # A record outside the runner's evidence-iter.json name carries no + # position in the run's order, so it can never be shown to be latest. + continue + produced_by = data.get("produced_by") + task_id = produced_by.get("task_id") if isinstance(produced_by, dict) else None + if not isinstance(task_id, str) or task_id not in entries: + continue + if task_id not in latest or (iteration_id, record_path.name) > ( + latest[task_id][0], latest[task_id][1].name): + latest[task_id] = (iteration_id, record_path, data) + for task_id, (_iteration_id, record_path, data) in sorted(latest.items()): + verified_by = data.get("verified_by") + recorded = verified_by.get("policy_digest") if isinstance(verified_by, dict) else None + if not isinstance(recorded, str): + continue + try: + live = verification_policy_digest(entries[task_id]) + except ChainHashError: + continue # a non-canonicalizable task entry is TASKS.json's own defect + if live != recorded: + issues.append(ContractIssue( + "policy_digest_mismatch", + f"{record_path.name}: the goalpost recorded for task {task_id!r} " + f"({recorded}) is not the live TASKS.json goalpost ({live}) — " + f"the declared verify/criterion_ref/depends_on/id changed after this " + f"verification; re-verify to record the current goalpost", + record_path)) + + def _validate_evidence_records(paths: LoopPaths, mode: str, issues: list[dict]) -> bool: - """Validate declared evidence@1 records and surface declared self-verification. + """Validate declared evidence@1 records, hash-verify what they reference, and + surface declared self-verification. Declared location: `.loop/evidence/*.json` (repo-os-contract.md §17). An absent directory is a no-op, so a contract with no evidence produces a byte-identical report — the same rule §22 pins for an absent event store. """ - from .evidence import evidence_issues # local: loop.evidence imports this module + from .evidence import evidence_issues, verify_evidence # local: loop.evidence imports this module _orphan_bundle_issues(paths, issues) evidence_dir = paths.loop_dir / EVIDENCE_DIR_NAME if not evidence_dir.is_dir(): return False checked = False + verifiable: list[tuple[int | None, Path, dict]] = [] for record_path in sorted(evidence_dir.glob("*.json")): data = _read_json(record_path, issues) if data is None: continue checked = True - for issue in evidence_issues(data, resolved_mode=mode): + record_issues = evidence_issues(data, resolved_mode=mode) + for issue in record_issues: issues.append(ContractIssue(issue["code"], f"{record_path.name}: {issue['message']}", record_path)) + if not record_issues: + # Guarded: verify_evidence() re-runs validate_evidence internally, so an + # unconditional call would report a malformed record twice. + result = verify_evidence(data, workspace_root=paths.workspace) + for issue in result["issues"]: + issues.append(ContractIssue(issue["code"], f"{record_path.name}: {issue['message']}", record_path)) + match = _EVIDENCE_RECORD_RE.fullmatch(record_path.name) + verifiable.append((int(match.group(1)) if match else None, record_path, data)) if _self_verified(data): issues.append(ContractIssue( "self_verified_evidence", f"{record_path.name}: produced_by.executor == verified_by.by " f"({data['produced_by']['executor']!r}) — the producer declares it verified its own work", record_path)) + _policy_digest_issues(paths, verifiable, issues) return checked diff --git a/scripts/test_doctor_evidence.py b/scripts/test_doctor_evidence.py index 0475e50..be04abe 100644 --- a/scripts/test_doctor_evidence.py +++ b/scripts/test_doctor_evidence.py @@ -14,10 +14,17 @@ def _codes(report): return {issue["code"] for issue in report["issues"]} +_BUNDLE_TEXT = json.dumps({"outcome": "PASS", "passed": True}) +_BUNDLE_SHA = hashlib.sha256(_BUNDLE_TEXT.encode("utf-8")).hexdigest() +# A legacy bundle name: doctor now hash-verifies a record's uri, so the file must +# exist, and a runner-shaped verify-iter.json would need its own record. +_RECORD_BUNDLE_NAME = "verify-T1.json" + + def _record(executor="worker-a", by="ci", **overrides): record = { "schema": "loop-engineer/evidence@1", "id": "e1", "kind": "verify-bundle", - "uri": ".loop/artifacts/verify-iter5.json", "sha256": "c" * 64, + "uri": f".loop/artifacts/{_RECORD_BUNDLE_NAME}", "sha256": _BUNDLE_SHA, "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", "produced_by": {"run_id": "run-1", "task_id": "T-1", "attempt": 1, "executor": executor}, "verified_by": {"by": by, "at": "2026-07-25T00:00:00+00:00", @@ -32,17 +39,20 @@ def _record(executor="worker-a", by="ci", **overrides): def _ws(tmp_path, records=(), bundles=(), name="workspace"): target = tmp_path / name scaffold(target) + artifacts = target / ".loop" / "artifacts" if records: directory = target / ".loop" / "evidence" directory.mkdir(parents=True, exist_ok=True) for index, record in enumerate(records): text = record if isinstance(record, str) else json.dumps(record) (directory / f"evidence-iter{index}.json").write_text(text, encoding="utf-8") + # Every _record() names this uri, so it must exist with exactly the bytes + # the record commits to. + artifacts.mkdir(parents=True, exist_ok=True) + (artifacts / _RECORD_BUNDLE_NAME).write_text(_BUNDLE_TEXT, encoding="utf-8") for bundle_name in bundles: - directory = target / ".loop" / "artifacts" - directory.mkdir(parents=True, exist_ok=True) - (directory / bundle_name).write_text( - json.dumps({"outcome": "PASS", "passed": True}), encoding="utf-8") + artifacts.mkdir(parents=True, exist_ok=True) + (artifacts / bundle_name).write_text(_BUNDLE_TEXT, encoding="utf-8") return target diff --git a/scripts/test_doctor_evidence_verification.py b/scripts/test_doctor_evidence_verification.py new file mode 100644 index 0000000..ddb32a9 --- /dev/null +++ b/scripts/test_doctor_evidence_verification.py @@ -0,0 +1,124 @@ +"""Doctor hash-verifies discovered records and compares their policy digest to TASKS.json.""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop.contract import doctor_report +from loop.scaffold import scaffold +from loop.verifier import verification_policy_digest + +try: # the repo's fallback-mode convention + import jsonschema # noqa: F401 + _HAS_JSONSCHEMA = True +except ImportError: # pragma: no cover - exercised in the fallback leg + _HAS_JSONSCHEMA = False + +_MODES = [pytest.param("basic"), + pytest.param("strict", marks=pytest.mark.skipif(not _HAS_JSONSCHEMA, + reason="jsonschema not installed"))] + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +_TASK = {"id": "T-1", "title": "t", "status": "pending", "criterion_ref": "C-1", + "verify": "./scripts/verify-fast.sh", "depends_on": [], "attempts": 0, "evidence": None} + + +def _ws(tmp_path, *, records, bundle_text='{"outcome": "PASS", "passed": true}', tasks=(_TASK,)): + target = tmp_path / "workspace" + scaffold(target) + (target / "TASKS.json").write_text(json.dumps( + {"schema": "loop-engineer/tasks@1", "project": "p", "tasks": list(tasks)}), encoding="utf-8") + # scaffold ships scripts/verify-fast without an extension; _TASK declares the + # .sh form, and a path-shaped task verify must resolve under the workspace. + (target / "scripts" / "verify-fast.sh").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + (target / ".loop" / "artifacts").mkdir(parents=True, exist_ok=True) + (target / ".loop" / "artifacts" / "verify-iter1.json").write_text(bundle_text, encoding="utf-8") + directory = target / ".loop" / "evidence" + directory.mkdir(parents=True, exist_ok=True) + for iteration_id, record in records: + (directory / f"evidence-iter{iteration_id}.json").write_text( + json.dumps(record), encoding="utf-8") + return target + + +def _record(*, iteration_id=1, policy_digest=None, uri=".loop/artifacts/verify-iter1.json", + sha256=None, task_id="T-1"): + text = '{"outcome": "PASS", "passed": true}' + return { + "schema": "loop-engineer/evidence@1", "id": f"run-1:{iteration_id}:verify", + "kind": "verify-bundle", "uri": uri, + "sha256": sha256 or hashlib.sha256(text.encode("utf-8")).hexdigest(), + "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", + "produced_by": {"run_id": "run-1", "task_id": task_id, "attempt": 1, + "executor": "worker-a"}, + "verified_by": {"by": "ci", "at": "2026-07-25T00:00:00+00:00", + "command": "./scripts/verify-fast.sh", "code_digest": None, + "code_digest_basis": "path_lookup", + "policy_digest": policy_digest or verification_policy_digest(_TASK)}, + } + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_hash_verified_record_is_clean(tmp_path, mode): + target = _ws(tmp_path, records=[(1, _record())]) + assert doctor_report(target, mode=mode)["ok"] is True + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_swapped_bundle_fails_doctor_with_hash_mismatch(tmp_path, mode): + target = _ws(tmp_path, records=[(1, _record())]) + (target / ".loop" / "artifacts" / "verify-iter1.json").write_text("{}", encoding="utf-8") + assert "hash_mismatch" in _codes(doctor_report(target, mode=mode)) + + +def test_a_record_whose_uri_is_absent_reports_missing_evidence_path(tmp_path): + target = _ws(tmp_path, records=[(1, _record(uri=".loop/artifacts/gone.json"))]) + assert "missing_evidence_path" in _codes(doctor_report(target)) + + +def test_a_record_whose_uri_escapes_the_workspace_reports_workspace_escape(tmp_path): + target = _ws(tmp_path, records=[(1, _record(uri="../escape.json"))]) + assert _codes(doctor_report(target)) & {"workspace_escape", "missing_evidence_path"} + + +@pytest.mark.parametrize("mode", _MODES) +def test_policy_digest_mismatch_against_the_live_task(tmp_path, mode): + moved = dict(_TASK, verify="true") + target = _ws(tmp_path, records=[(1, _record())], tasks=(moved,)) + assert "policy_digest_mismatch" in _codes(doctor_report(target, mode=mode)) + + +def test_policy_digest_agreement_is_silent(tmp_path): + target = _ws(tmp_path, records=[(1, _record())]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_only_the_latest_record_per_task_is_compared(tmp_path): + stale = _record(iteration_id=1, policy_digest="d" * 64) + current = _record(iteration_id=2) + target = _ws(tmp_path, records=[(1, stale), (2, current)]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_a_record_naming_an_unknown_task_is_not_compared(tmp_path): + target = _ws(tmp_path, records=[(1, _record(task_id="T-404", policy_digest="d" * 64))]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_a_record_without_a_policy_digest_is_not_compared(tmp_path): + record = _record() + record["verified_by"]["policy_digest"] = None + target = _ws(tmp_path, records=[(1, record)]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +@pytest.mark.parametrize("mode", _MODES) +def test_evidence_schema_id_still_joins_schemas_checked(tmp_path, mode): + target = _ws(tmp_path, records=[(1, _record())]) + assert "loop-engineer/evidence@1" in doctor_report(target, mode=mode)["schemas_checked"] From 0d1bd246e082746a2f9f2e11c4748a3650125f29 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 15:28:34 -0400 Subject: [PATCH 06/22] fix(doctor): fail closed when a cited task's goalpost cannot be canonicalized --- loop/contract.py | 17 ++++++++++--- scripts/test_doctor_evidence_verification.py | 25 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/loop/contract.py b/loop/contract.py index 7927cb0..a1eff41 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -694,7 +694,9 @@ def _policy_digest_issues( goalposts that were current when they were written, and comparing them would make every honest re-verification a permanent failure (repo-os-contract.md §17). A record naming a task that is no longer in TASKS.json is not compared at all — - a renamed or removed task is a replan, not a forgery. + a renamed or removed task is a replan, not a forgery. A task that is present but + cannot be canonicalized is reported rather than skipped: a comparison that cannot + run has not passed. """ from .verifier import verification_policy_digest @@ -722,8 +724,17 @@ def _policy_digest_issues( continue try: live = verification_policy_digest(entries[task_id]) - except ChainHashError: - continue # a non-canonicalizable task entry is TASKS.json's own defect + except ChainHashError as exc: + # Fail closed. Structural-fallback mode carries no tasks@1 type-check, so + # skipping here would leave a stale goalpost reported by nothing at all. + issues.append(ContractIssue( + "policy_digest_mismatch", + f"{record_path.name}: the goalpost recorded for task {task_id!r} " + f"({recorded}) cannot be compared against the live TASKS.json entry, " + f"because that entry is not canonicalizable ({exc}) — agreement is " + f"unestablished, and unestablished is not agreement", + record_path)) + continue if live != recorded: issues.append(ContractIssue( "policy_digest_mismatch", diff --git a/scripts/test_doctor_evidence_verification.py b/scripts/test_doctor_evidence_verification.py index ddb32a9..5a159f9 100644 --- a/scripts/test_doctor_evidence_verification.py +++ b/scripts/test_doctor_evidence_verification.py @@ -118,6 +118,31 @@ def test_a_record_without_a_policy_digest_is_not_compared(tmp_path): assert "policy_digest_mismatch" not in _codes(doctor_report(target)) +def _nan_task(): + """A live entry json.loads accepts but verification_policy_digest cannot canonicalize.""" + return dict(_TASK, criterion_ref=float("nan")) + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_non_canonicalizable_live_task_fails_closed_in_both_modes(tmp_path, mode): + """An unrunnable comparison is not an agreement. + + jsonschema mode independently reports the malformed entry as a schema_violation, + but structural-fallback has no tasks@1 type-check — skipping here left a stale + goalpost reported by nothing, and doctor answered ok. + """ + target = _ws(tmp_path, records=[(1, _record(policy_digest="d" * 64))], + tasks=(_nan_task(),)) + report = doctor_report(target, mode=mode) + assert "policy_digest_mismatch" in _codes(report) + assert report["ok"] is False + + +def test_a_non_canonicalizable_task_no_record_names_is_not_a_goalpost_finding(tmp_path): + target = _ws(tmp_path, records=[(1, _record(task_id="T-404"))], tasks=(_nan_task(),)) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + @pytest.mark.parametrize("mode", _MODES) def test_evidence_schema_id_still_joins_schemas_checked(tmp_path, mode): target = _ws(tmp_path, records=[(1, _record())]) From afd7650b3509c8c368c999f8a79640b0de4773fc Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 16:13:38 -0400 Subject: [PATCH 07/22] =?UTF-8?q?feat(completion):=20all=5Frequired=5Fveri?= =?UTF-8?q?fied=5Fevidence=20=E2=80=94=20Succeeded=20on=20bound,=20hash-ve?= =?UTF-8?q?rified=20records?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the second completion mode. Under all_required_verified_evidence a Succeeded terminal may only cite evidence@1 records that are self-consistent (the bundle hashes to the digest the record declares), bound into the event chain at their CURRENT bytes, and in agreement with the live TASKS.json goalpost. Hash-verification alone is forgeable by anything with .loop/ write access, so the two layers that hold a workspace (emit.terminate, doctor) enforce all three sub-checks through ONE definition — contract._strict_evidence_failure — while the two pure layers (the reducer fold and integrations.to_terminal_state) keep only the structural half rather than pretend to verify what they cannot reach. An unreadable event store is a refusal, never a skip. Three changes reach beyond the bar itself and are called out deliberately: * loop/simulate.py — _predict no longer restates the auto-terminal payload literally. That literal was a second definition of the very thing this change edits, and the predicted-equals-real pin (test_loop_simulate_cli.py:101) is what would have caught the drift only after it shipped. It now builds the payload with the runner's own runner._auto_terminal_payload. Same anti-drift rule this slice makes load-bearing for _strict_evidence_failure, applied to the payload. * scripts/test_loop_simulate_cli.py:86 — the auto-terminal snapshot gains the "reason" key the payload genuinely now carries. Whole-dict equality is preserved on purpose: the assertion is not relaxed to a subset check, and the reason is not dropped, because a silently weaker policy with no stated cause is the downgrade this mode exists to forbid. * loop/runner.py — the auto-terminal now adopts the strict mode honestly. It evaluates the same shared predicate against every candidate record BEFORE it chooses the mode and BEFORE it appends terminal_written. Previously it adopted the mode on evidence FILE EXISTENCE and let emit.terminate apply the real bar afterwards, which both overstated the bar and left the event durable before raising — the committed-then- refused seam this slice removes. When the bar is not met the payload stays all_required with evidence ["RUNLOG.md"] and a reason that names why, distinguishing a task that produced no record at all from a record present but failing a sub-check. Suite: 1211 passed / 16 skipped (pyyaml+jsonschema), 1117 passed / 110 skipped (pyyaml only), each with the one governed intermediate red owned by the verifier- identity task. Interpreter matrix 3.12 and 3.13 green. Both dogfood examples stay doctor-clean. --- loop/completion.py | 25 +- loop/contract.py | 112 ++++++ loop/emit.py | 29 ++ loop/integrations.py | 8 + loop/reducer.py | 11 +- loop/runner.py | 78 ++++- loop/simulate.py | 11 +- schemas/terminal.schema.json | 5 +- scripts/test_completion_verified_evidence.py | 346 +++++++++++++++++++ scripts/test_loop_simulate_cli.py | 2 +- 10 files changed, 607 insertions(+), 20 deletions(-) create mode 100644 scripts/test_completion_verified_evidence.py diff --git a/loop/completion.py b/loop/completion.py index 6d9a1ba..62fcca7 100644 --- a/loop/completion.py +++ b/loop/completion.py @@ -11,9 +11,15 @@ from collections.abc import Mapping from typing import Final, Literal, TypeAlias, cast -CompletionMode: TypeAlias = Literal["all_required"] +CompletionMode: TypeAlias = Literal["all_required", "all_required_verified_evidence"] DEFAULT_COMPLETION_MODE: Final[CompletionMode] = "all_required" -SUPPORTED_COMPLETION_MODES: Final[tuple[CompletionMode, ...]] = (DEFAULT_COMPLETION_MODE,) +VERIFIED_EVIDENCE_MODE: Final[CompletionMode] = "all_required_verified_evidence" +SUPPORTED_COMPLETION_MODES: Final[tuple[CompletionMode, ...]] = ( + DEFAULT_COMPLETION_MODE, + VERIFIED_EVIDENCE_MODE, +) + +_RECORD_PREFIX = ".loop/evidence/" class CompletionPolicyError(ValueError): @@ -63,11 +69,24 @@ def criteria_satisfy_completion( ``True``; truthy substitutes such as ``1`` are deliberately rejected. """ normalized = normalize_completion_policy(policy) - if normalized["mode"] == "all_required": + if normalized["mode"] in SUPPORTED_COMPLETION_MODES: + # The criteria half is deliberately shared: the verified-evidence mode raises + # the EVIDENCE bar, never the criteria bar. return bool(criteria_met) and all(value is True for value in criteria_met.values()) raise AssertionError(f"unhandled completion policy: {normalized!r}") +def policy_requires_verified_evidence(policy: object | None = None) -> bool: + """True when this policy's evidence bar is hash-verified records, not path strings.""" + return normalize_completion_policy(policy)["mode"] == VERIFIED_EVIDENCE_MODE + + +def evidence_entry_is_record_shaped(entry: object) -> bool: + """The structural half of the bar — the most a pure, I/O-free layer can honestly check.""" + return (isinstance(entry, str) and entry.startswith(_RECORD_PREFIX) + and entry.endswith(".json") and ".." not in entry) + + def unmet_required_criteria(criteria_met: Mapping[str, object]) -> tuple[str, ...]: """Return stable string identifiers for criteria not proven true.""" return tuple(sorted(key for key, value in criteria_met.items() if value is not True)) diff --git a/loop/contract.py b/loop/contract.py index a1eff41..fbda791 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import re from pathlib import Path @@ -8,9 +9,11 @@ from . import fsm from .chain import ChainHashError from .completion import ( + VERIFIED_EVIDENCE_MODE, CompletionPolicyError, criteria_satisfy_completion, normalize_completion_policy, + policy_requires_verified_evidence, unmet_required_criteria, ) from .paths import ARTIFACTS_DIR_NAME, EVIDENCE_DIR_NAME, LoopPaths, resolve_loop_paths @@ -787,6 +790,113 @@ def _validate_evidence_records(paths: LoopPaths, mode: str, issues: list[dict]) return checked +def _strict_evidence_failure(entry: object, paths: LoopPaths, + bound: Mapping[str, str] | None) -> str | None: + """The ONE definition of the verified-evidence bar (plan decision 14). + + Returns ``None`` when the cited entry can back ``Succeeded``, otherwise a detail + string naming which of the three sub-checks failed: + + 1. self-consistency — the entry is a readable evidence@1 record whose ``uri`` + resolves inside the workspace and hashes to the digest it declares; + 2. chain-boundness — some event bound this record's path AT its current bytes. + ``bound is None`` means there is no event store at all, and the sub-check is + skipped: the store-less writer-API path is a DOCUMENTED degradation, not a pass; + 3. goalpost agreement — when the record names a task still in TASKS.json, its + recorded ``policy_digest`` equals the live one. A goalpost that cannot even be + computed is a FAILURE, not a skip: unestablished is not agreement (R007). + + ``emit.terminate`` imports this function rather than restating it. Two hand-written + copies of a three-part security check drift, and a drift here is a silent false + completion. + """ + from .evidence import validate_evidence, verify_evidence # local: loop.evidence imports this module + from .verifier import verification_policy_digest + + if not isinstance(entry, str) or not entry.strip(): + return "is not a workspace-relative evidence record path" + try: + blob = (paths.workspace / entry).read_bytes() + except (OSError, ValueError) as exc: + return f"is not verified evidence: the record cannot be read ({exc})" + try: + record = json.loads(blob.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return f"is not verified evidence: the record is not UTF-8 JSON ({exc})" + if not isinstance(record, dict): + return "is not verified evidence: the record is not a JSON object" + validation = validate_evidence(record, mode="basic") + if not validation["ok"]: + return f"is not verified evidence: {[issue['message'] for issue in validation['issues']]}" + verified = verify_evidence(record, workspace_root=paths.workspace) + if not verified["ok"]: + return f"is not verified evidence: {[issue['message'] for issue in verified['issues']]}" + + if bound is not None: + committed = bound.get(entry) + if committed is None: + return ("is not bound into the event chain — no event committed this record's " + "digest, so no dispatch produced it") + current = hashlib.sha256(blob).hexdigest() + if committed != current: + return (f"is bound at a different digest: the chain committed {committed}, " + f"the record now hashes to {current}") + + produced_by = record.get("produced_by") + task_id = produced_by.get("task_id") if isinstance(produced_by, dict) else None + if not isinstance(task_id, str): + return None + declared = _read_json(paths.tasks, []) # already reported by the contract read + entries = declared.get("tasks") if isinstance(declared, dict) else None + live = next((task for task in (entries if isinstance(entries, list) else []) + if isinstance(task, dict) and task.get("id") == task_id), None) + if live is None: + # A renamed or removed task is a replan, not a forgery (decision 5). + return None + verified_by = record.get("verified_by") + recorded = verified_by.get("policy_digest") if isinstance(verified_by, dict) else None + try: + current_goalpost = verification_policy_digest(live) + except ChainHashError as exc: + return (f"records a goalpost that cannot be compared against the live TASKS.json " + f"goalpost for task {task_id!r}: that entry is not canonicalizable ({exc}), " + f"so agreement is unestablished — and unestablished is not agreement") + if recorded != current_goalpost: + return (f"records a goalpost that is not the live TASKS.json goalpost for task " + f"{task_id!r}: recorded {recorded!r}, live {current_goalpost!r}") + return None + + +def _check_verified_evidence_terminal(terminal: Any, paths: LoopPaths, issues: list[dict]) -> None: + """Read-time twin of emit.terminate's strict bar — one predicate, two callers.""" + if not isinstance(terminal, dict) or terminal.get("state") != "Succeeded": + return + try: + if not policy_requires_verified_evidence(terminal.get("completion_policy")): + return + except CompletionPolicyError: + return # already reported by _validate_terminal / the terminal@1 schema + from .runtime import RuntimeStoreError, bound_artifact_digests # local: runtime imports this module + + try: + bound = bound_artifact_digests(paths.workspace) + except RuntimeStoreError as exc: + issues.append(ContractIssue( + "unverified_evidence_terminal", + f"the terminal declares {VERIFIED_EVIDENCE_MODE} but the event store could not " + f"be read ({exc}) — chain-boundness is unestablished, and unestablished is not " + f"proof", paths.terminal)) + return + evidence = terminal.get("evidence") + for entry in evidence if isinstance(evidence, list) else []: + detail = _strict_evidence_failure(entry, paths, bound) + if detail is not None: + issues.append(ContractIssue( + "unverified_evidence_terminal", + f"the terminal declares {VERIFIED_EVIDENCE_MODE} but {entry!r} {detail}", + paths.terminal)) + + def _validate_optional_records(paths: LoopPaths, mode: str, issues: list[dict]) -> set[str]: """Validate record files that are present; return the set of record schema keys actually checked (``repair``/``rollout``/``receipt``/``evidence``) so ``doctor`` can @@ -888,6 +998,8 @@ def validate_contract(target: str | Path, *, mode: str | None = None) -> dict[st _check_stub_verify_scripts(paths, issues) _check_verify_surface(paths, tasks, issues) records_checked = _validate_optional_records(paths, mode, issues) + if terminal is not None: + _check_verified_evidence_terminal(terminal, paths, issues) schemas_checked = list(SCHEMA_IDS) + [ schema_id for key, schema_id in _RECORD_SCHEMA_IDS if key in records_checked diff --git a/loop/emit.py b/loop/emit.py index 660772a..9318ee2 100644 --- a/loop/emit.py +++ b/loop/emit.py @@ -19,13 +19,16 @@ from . import fsm from .chain import ChainHashError from .completion import ( + VERIFIED_EVIDENCE_MODE, CompletionPolicyError, criteria_satisfy_completion, normalize_completion_policy, + policy_requires_verified_evidence, unmet_required_criteria, ) from .contract import ( TERMINAL_STATES, + _strict_evidence_failure, _validate_record, _validate_terminal, _validation_mode, @@ -243,6 +246,28 @@ def append_receipt( return receipts +def _refuse_unverified_evidence(paths, evidence: Sequence[str]) -> None: + """Enforce the verified-evidence bar at write time, through contract's one definition. + + An unreadable store is a REFUSAL, never a skip: the whole point of the mode is that + a cited record is bound into a chain, and a chain nobody can read proves nothing. + """ + from .runtime import RuntimeStoreError, bound_artifact_digests # local: runtime -> events -> emit + + try: + bound = bound_artifact_digests(paths.workspace) + except RuntimeStoreError as exc: + raise EmitError( + f"refusing Succeeded under {VERIFIED_EVIDENCE_MODE}: the event store could not be " + f"read ({exc}) — chain-boundness is unestablished, and unestablished is not proof" + ) from exc + for entry in evidence: + detail = _strict_evidence_failure(entry, paths, bound) + if detail is not None: + raise EmitError( + f"refusing Succeeded under {VERIFIED_EVIDENCE_MODE}: {entry} {detail}") + + def terminate( target: str | Path, *, @@ -303,6 +328,10 @@ def terminate( raise EmitError( "refusing Succeeded because not all required criteria are proven true: " + detail ) + if policy_requires_verified_evidence(normalized_policy): + # Resolved here rather than by hoisting the assignment below, so the order in + # which terminate reports its refusals is exactly what it was. + _refuse_unverified_evidence(_require_contract(target), evidence) paths = _require_contract(target) terminal_path = paths.loop_dir / "terminal_state.json" diff --git a/loop/integrations.py b/loop/integrations.py index 8b5a736..55ff65c 100644 --- a/loop/integrations.py +++ b/loop/integrations.py @@ -28,7 +28,9 @@ from .completion import ( CompletionPolicyError, criteria_satisfy_completion, + evidence_entry_is_record_shaped, normalize_completion_policy, + policy_requires_verified_evidence, unmet_required_criteria, ) @@ -169,6 +171,12 @@ def body(state: str, reason: str) -> dict: ) if evidence_error is not None: return body("FailedUnverifiable", "invalid evidence artifacts: " + evidence_error) + # Structural half only, fail-closed: a pure projection holds no workspace and must + # not pretend to hash-verify or chain-check what it cites. + if (policy_requires_verified_evidence(normalized_policy) + and not all(evidence_entry_is_record_shaped(item) for item in artifacts)): + return body("FailedUnverifiable", + "green gate but an evidence artifact is not verified evidence — cannot certify") if not artifacts: return body("FailedUnverifiable", "green gate but no evidence artifacts — cannot certify") if not outcome.reached_end: diff --git a/loop/reducer.py b/loop/reducer.py index e88ef7e..4860716 100644 --- a/loop/reducer.py +++ b/loop/reducer.py @@ -5,7 +5,9 @@ from typing import Any, Iterable, Mapping from . import chain, fsm -from .completion import CompletionPolicyError, criteria_satisfy_completion, normalize_completion_policy +from .completion import (CompletionPolicyError, criteria_satisfy_completion, + evidence_entry_is_record_shaped, normalize_completion_policy, + policy_requires_verified_evidence) from .contract import TERMINAL_STATES from .events import _structural_validate_event @@ -47,6 +49,13 @@ def _validate_terminal_payload_semantics(payload: Mapping[str, Any]) -> None: evidence = payload.get("evidence") if not isinstance(evidence, list) or not evidence: raise EventReplayError("Succeeded terminal has empty evidence (G1)") + # The STRUCTURAL half only: this fold is pure and holds no workspace, so it can + # check the shape of a citation and nothing more (repo-os-contract §16). + if (policy_requires_verified_evidence(policy) + and not all(evidence_entry_is_record_shaped(item) for item in evidence)): + raise EventReplayError( + "Succeeded terminal declares verified evidence but an entry is not a " + ".loop/evidence record (G1)") def _validate_superseded_payload_semantics(payload: Mapping[str, Any]) -> None: diff --git a/loop/runner.py b/loop/runner.py index 9d1c29c..daa32a5 100644 --- a/loop/runner.py +++ b/loop/runner.py @@ -11,11 +11,13 @@ from typing import Any, Callable from . import emit +from .completion import DEFAULT_COMPLETION_MODE, VERIFIED_EVIDENCE_MODE +from .contract import _strict_evidence_failure from .events import (EventRowDecodeError, EventStoreOperationalError, SQLiteEventStore, read_event_rows, validate_event) -from .paths import resolve_loop_paths +from .paths import EVIDENCE_DIR_NAME, resolve_loop_paths from .reducer import ChainBreakError, reduce_events -from .runtime import RuntimeStoreError, _read_store +from .runtime import RuntimeStoreError, _read_store, bound_artifact_digests from .verifier import executed_verifier_identity, injected_verifier_identity @@ -175,6 +177,70 @@ def _reconcile_legacy_iteration(target: str | Path, projection: dict[str, Any]) current_id = iteration_id +def _evidence_records_by_task(paths: Any, projection: dict[str, Any]) -> dict[str, list[str]]: + """Map each task id to the evidence records this run's passing iterations left on disk.""" + records: dict[str, list[str]] = {} + for entry in projection.get("runlog_entries", []): + if entry.get("outcome") != "task_passed": + continue + task_id, iteration_id = entry.get("task_id"), entry.get("iteration_id") + if not isinstance(task_id, str) or not isinstance(iteration_id, int): + continue + record = paths.loop_dir / EVIDENCE_DIR_NAME / f"evidence-iter{iteration_id}.json" + if record.is_file(): + records.setdefault(task_id, []).append(record.relative_to(paths.workspace).as_posix()) + return records + + +def _unmet_strict_bar(paths: Any, cited: list[str]) -> str | None: + """Why the cited records cannot back the strict mode, or ``None`` when they can. + + The SAME predicate ``emit.terminate`` enforces, evaluated against the very records + the payload would cite. Existence is not satisfaction: adopting the strict mode + because a file is present would overstate the bar (decision 7) and would re-open the + committed-then-refused seam this slice closes (decision 2), because the writer runs + the predicate afterwards and raises once the event is already durable. + """ + try: + bound = bound_artifact_digests(paths.workspace) + except RuntimeStoreError as exc: + return (f"the event store could not be read ({exc}), so chain-boundness is " + f"unestablished — and unestablished is not proof") + for entry in cited: + detail = _strict_evidence_failure(entry, paths, bound) + if detail is not None: + return f"{entry} {detail}" + return None + + +def _auto_terminal_payload(paths: Any, tasks: list[dict], projection: dict[str, Any]) -> dict[str, Any]: + """Adopt the verified-evidence mode only when this run can honestly satisfy it. + + Choosing the weaker policy silently would be a self-serving downgrade, so the + all_required branch always says WHY — distinguishing a task that produced no record + at all from a record that is present but fails the bar. + """ + payload: dict[str, Any] = { + "state": "Succeeded", "criteria_met": {task["id"]: True for task in tasks}, + "evidence": ["RUNLOG.md"], "false_completion": False, + "completion_policy": {"mode": DEFAULT_COMPLETION_MODE}, + "iteration_id": projection["iteration_id"], + } + records = _evidence_records_by_task(paths, projection) + missing = sorted(str(task.get("id")) for task in tasks if task.get("id") not in records) + if missing: + payload["reason"] = "tasks with no evidence record: " + ", ".join(missing) + return payload + cited = sorted({record for task in tasks for record in records[task["id"]]}) + unmet = _unmet_strict_bar(paths, cited) + if unmet is not None: + payload["reason"] = "evidence records do not meet the verified-evidence bar: " + unmet + return payload + payload["evidence"] = cited + payload["completion_policy"] = {"mode": VERIFIED_EVIDENCE_MODE} + return payload + + def _reconcile_legacy_terminal(target: str | Path, projection: dict[str, Any]) -> None: """Replay the terminal's already-recorded payload into existing emit APIs.""" terminal = projection.get("terminal") @@ -182,11 +248,13 @@ def _reconcile_legacy_terminal(target: str | Path, projection: dict[str, Any]) - return paths = resolve_loop_paths(target) if not paths.terminal.exists(): + reason = terminal.get("reason") emit.terminate( target, state=terminal["state"], criteria_met=terminal["criteria_met"], evidence=terminal["evidence"], + reason=reason if isinstance(reason, str) else "", false_completion=terminal["false_completion"], iteration_id=terminal.get("iteration_id"), completion_policy=terminal.get("completion_policy"), @@ -220,11 +288,7 @@ def dispatch_once( done = done_task_ids(tasks, projection) if all(task.get("id") in done for task in tasks): iteration_id = projection["iteration_id"] - payload = { - "state": "Succeeded", "criteria_met": {task["id"]: True for task in tasks}, - "evidence": ["RUNLOG.md"], "false_completion": False, - "completion_policy": {"mode": "all_required"}, "iteration_id": iteration_id, - } + payload = _auto_terminal_payload(paths, tasks, projection) store = SQLiteEventStore(paths.loop_dir / "events.db") _store_append(store, run_id, "terminal_written", payload, actor="loop.run", expected_sequence=projection["last_sequence"] + 1) diff --git a/loop/simulate.py b/loop/simulate.py index d5eac40..27827d6 100644 --- a/loop/simulate.py +++ b/loop/simulate.py @@ -67,13 +67,10 @@ def _predict(paths: Any, projection: dict[str, Any]) -> dict[str, Any]: if task is None: done = runner.done_task_ids(tasks, projection) if all(candidate.get("id") in done for candidate in tasks): - payload = { - "state": "Succeeded", - "criteria_met": {candidate["id"]: True for candidate in tasks}, - "evidence": ["RUNLOG.md"], "false_completion": False, - "completion_policy": {"mode": "all_required"}, - "iteration_id": projection["iteration_id"], - } + # Built by the runner's own function, never restated here: a second copy of + # the auto-terminal payload silently breaks the predicted-equals-real pin + # (test_simulate_predicted_terminal_payload_matches_real_..._exactly). + payload = runner._auto_terminal_payload(paths, tasks, projection) return {**empty, "action": "would_write_terminal", "predicted_terminal": payload} return {**empty, "action": "would_block"} diff --git a/schemas/terminal.schema.json b/schemas/terminal.schema.json index db3ecb9..5a6e57c 100644 --- a/schemas/terminal.schema.json +++ b/schemas/terminal.schema.json @@ -77,7 +77,10 @@ ], "properties": { "mode": { - "const": "all_required" + "enum": [ + "all_required", + "all_required_verified_evidence" + ] } }, "additionalProperties": false, diff --git a/scripts/test_completion_verified_evidence.py b/scripts/test_completion_verified_evidence.py new file mode 100644 index 0000000..1b7fbb2 --- /dev/null +++ b/scripts/test_completion_verified_evidence.py @@ -0,0 +1,346 @@ +"""all_required_verified_evidence: the same bar at four layers, each as strong as it can be. + +Hash-verification ALONE is a forgeable bar: a worker with `.loop/` write access can +hand-write a bundle, its digest, a matching record and a correct policy_digest with +zero dispatch and zero events. The two layers that hold a workspace therefore also +require chain-boundness and live-goalpost agreement (plan decision 14); the two pure +layers keep only the structural half, because pretending otherwise is the failure +this mode exists to close. +""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop import emit, integrations +from loop.completion import (VERIFIED_EVIDENCE_MODE, normalize_completion_policy, + policy_requires_verified_evidence) +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.reducer import EventReplayError, reduce_events +from loop.runner import dispatch_once +from loop.verifier import verification_policy_digest + +try: # the repo's fallback-mode convention + import jsonschema # noqa: F401 + _HAS_JSONSCHEMA = True +except ImportError: # pragma: no cover - exercised in the fallback leg + _HAS_JSONSCHEMA = False + +_MODES = [pytest.param("basic"), + pytest.param("strict", marks=pytest.mark.skipif(not _HAS_JSONSCHEMA, + reason="jsonschema not installed"))] + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + + +def _task(task_id, deps=(), status="pending", **extra): + return {"id": task_id, "title": task_id, "status": status, "criterion_ref": task_id, + "verify": _VERIFY, "depends_on": list(deps), "attempts": 0, "evidence": None, **extra} + + +def _ready(tmp_path, tasks): + """A contract projected to execute-task with the iteration counter still at 0. + + Same shape as scripts/test_evidence_binding_runner.py: the ramp records the FSM + walk, not work, so the first dispatch is iteration 1 and every artifact path + asserted below is literal. + """ + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": tasks}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + return workspace + + +@pytest.fixture +def ready_workspace(tmp_path): + return _ready(tmp_path, [_task("T-1")]) + + +@pytest.fixture +def dispatched_workspace(tmp_path): + """One real dispatch: `.loop/evidence/evidence-iter1.json` exists and IS chain-bound.""" + workspace = _ready(tmp_path, [_task("T-1")]) + dispatch_once(workspace) + return workspace + + +@pytest.fixture +def predone_workspace(tmp_path): + """Every task already declaratively done, so the auto-terminal fires with zero records.""" + return _ready(tmp_path, [_task("T-1", status="done", evidence="RUNLOG.md")]) + + +def test_the_new_mode_is_supported_and_normalizes(): + assert normalize_completion_policy(VERIFIED_EVIDENCE_MODE) == {"mode": VERIFIED_EVIDENCE_MODE} + assert policy_requires_verified_evidence({"mode": VERIFIED_EVIDENCE_MODE}) is True + + +def test_legacy_null_policy_still_normalizes_to_all_required(): + assert normalize_completion_policy(None) == {"mode": "all_required"} + assert policy_requires_verified_evidence(None) is False + + +def test_terminate_refuses_unverified_evidence_under_the_new_mode(dispatched_workspace): + with pytest.raises(emit.EmitError, match="verified evidence"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=["RUNLOG.md"], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_terminate_accepts_hash_verified_evidence_under_the_new_mode(dispatched_workspace): + path = emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} + + +def test_terminate_under_the_default_mode_is_unchanged(dispatched_workspace): + path = emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=["RUNLOG.md"]) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": "all_required"} + + +@pytest.mark.parametrize("mode", _MODES) +def test_doctor_reports_unverified_evidence_terminal(dispatched_workspace, mode): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + (dispatched_workspace / ".loop" / "artifacts" / "verify-iter1.json").write_text( + "{}", encoding="utf-8") + assert "unverified_evidence_terminal" in _codes(doctor_report(dispatched_workspace, mode=mode)) + + +# --- decision 14: hash-verified is NOT enough when the layer can reach further --- + +def _handwritten_record(workspace, *, name="evidence-iter9.json", task_id="T-1"): + """A self-consistent, never-dispatched record: real bundle, real digest, real policy.""" + bundle = workspace / ".loop" / "artifacts" / "verify-iter9.json" + bundle.parent.mkdir(parents=True, exist_ok=True) + text = '{"outcome": "PASS", "passed": true}' + bundle.write_text(text, encoding="utf-8") + tasks = json.loads((workspace / "TASKS.json").read_text(encoding="utf-8")) + entry = next(t for t in tasks["tasks"] if t["id"] == task_id) + record = { + "schema": "loop-engineer/evidence@1", "id": "hand:9:verify", "kind": "verify-bundle", + "uri": ".loop/artifacts/verify-iter9.json", + "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), + "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", + "produced_by": {"run_id": "run-1", "task_id": task_id, "attempt": 1, + "executor": "worker-a"}, + "verified_by": {"by": "ci", "at": "2026-07-25T00:00:00+00:00", + "command": entry["verify"], "code_digest": None, + "code_digest_basis": "path_lookup", + "policy_digest": verification_policy_digest(entry)}, + } + path = workspace / ".loop" / "evidence" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return f".loop/evidence/{name}" + + +def test_terminate_refuses_a_hash_verified_but_unbound_record_when_a_store_exists( + dispatched_workspace): + """The BLOCKER case: self-consistency is not dispatch. No event bound this record.""" + entry = _handwritten_record(dispatched_workspace) + with pytest.raises(emit.EmitError, match="not bound"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[entry], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_terminate_refuses_a_cited_record_whose_goalpost_moved_or_cannot_be_computed( + dispatched_workspace): + """Per-cited-record, deliberately stricter than doctor's latest-per-task rule. + + Both halves live in one test because the acceptance count is fixed at 17: a moved + goalpost and an UNCOMPUTABLE one are the same refusal, since a comparison that + cannot run has not passed (R007 — an errored check fails, it never skips). + """ + tasks_path = dispatched_workspace / "TASKS.json" + data = json.loads(tasks_path.read_text(encoding="utf-8")) + data["tasks"][0]["verify"] = "true" + tasks_path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(emit.EmitError, match="goalpost"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + + data["tasks"][0]["criterion_ref"] = float("nan") # json.loads accepts it; canonical JSON does not + tasks_path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(emit.EmitError, match="goalpost"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_doctor_reports_an_unbound_record_cited_by_a_strict_terminal(dispatched_workspace): + """Read-time twin of the write-time refusal — same predicate, one definition. + + The terminal file is hand-written, not emitted: emit.terminate REFUSES this exact + record (the test above), so the only way a strict terminal citing an unbound record + reaches disk is a worker writing it directly — which is the adversary being modelled. + """ + entry = _handwritten_record(dispatched_workspace) + (dispatched_workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "project": dispatched_workspace.name, + "state": "Succeeded", "criteria_met": {"C-1": True}, + "completion_policy": {"mode": VERIFIED_EVIDENCE_MODE}, "evidence": [entry], + "false_completion": False, "terminated_at": "2026-07-25T00:00:00+00:00", + "reason": "hand-written"}, indent=2, sort_keys=True) + "\n", encoding="utf-8") + assert "unverified_evidence_terminal" in _codes(doctor_report(dispatched_workspace)) + + +def _corrupt_store(workspace): + """Make the event store genuinely unreadable, at the bytes. + + Not a chmod: this suite runs as root in CI, where mode bits are advisory and the + 'unreadable' store would still open. Overwriting the header (and dropping the WAL + sidecars sqlite would otherwise recover from) is unreadable for everyone. + """ + loop_dir = workspace / ".loop" + for sidecar in ("events.db-wal", "events.db-shm"): + (loop_dir / sidecar).unlink(missing_ok=True) + (loop_dir / "events.db").write_bytes(b"this is not a sqlite database" * 64) + + +def test_terminate_refuses_the_new_mode_when_the_event_store_cannot_be_read(dispatched_workspace): + """Fail-closed, not fail-open: a chain nobody can read binds nothing (R007).""" + _corrupt_store(dispatched_workspace) + with pytest.raises(emit.EmitError, match="event store could not be read"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_doctor_reports_a_strict_terminal_whose_event_store_cannot_be_read(dispatched_workspace): + """The read-time twin degrades to a FINDING rather than an exception: doctor's job is + to report every issue it can see, so an unreadable store must not abort the sweep.""" + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + _corrupt_store(dispatched_workspace) + report = doctor_report(dispatched_workspace) # must not raise + assert "unverified_evidence_terminal" in _codes(report) + # Pinned to the store-read branch: the cited record itself is still perfectly valid, + # so any OTHER path to this code would mean the finding is right for the wrong reason. + assert any("event store could not be read" in issue["message"] + for issue in report["issues"] + if issue["code"] == "unverified_evidence_terminal") + + +def _terminal_event(evidence, mode): + return [{"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 0, "event_id": "e0", + "type": "contract_opened", "actor": "op", "ts": "2026-07-25T00:00:00+00:00", + "causation_id": None, "correlation_id": None, "payload": {"workspace": "ws"}, + "artifact_hashes": []}, + {"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 1, "event_id": "e1", + "type": "terminal_written", "actor": "op", "ts": "2026-07-25T00:00:01+00:00", + "causation_id": None, "correlation_id": None, "artifact_hashes": [], + "payload": {"state": "Succeeded", "criteria_met": {"C-1": True}, + "evidence": evidence, "false_completion": False, + "completion_policy": {"mode": mode}}}] + + +def test_the_reducer_refuses_a_non_record_evidence_entry_under_the_new_mode(): + with pytest.raises(EventReplayError, match="verified evidence"): + reduce_events(_terminal_event(["RUNLOG.md"], VERIFIED_EVIDENCE_MODE)) + + +def test_the_reducer_accepts_record_shaped_evidence_under_the_new_mode(): + projection = reduce_events( + _terminal_event([".loop/evidence/evidence-iter1.json"], VERIFIED_EVIDENCE_MODE)) + assert projection["terminal"]["state"] == "Succeeded" + + +def test_integrations_projection_refuses_non_record_evidence_under_the_new_mode(): + outcome = integrations.EngineOutcome(reached_end=True, artifacts=("RUNLOG.md",)) + body = integrations.to_terminal_state( + outcome, {"verdict": "Succeeded", "passed_visible": True, "passed_holdout": True, + "false_completion": False, + "visible": [{"id": "C-1", "passed": True}], + "holdout": [{"id": "H-1", "passed": True}]}, + # A clean anticheat result needs `downgrade_to` present, or the projection fails + # closed on the anticheat input and never reaches the evidence bar under test. + {"findings": [], "clean": True, "downgrade_to": None}, {"C-1": True}, + completion_policy=VERIFIED_EVIDENCE_MODE) + assert body["state"] == "FailedUnverifiable" and "verified evidence" in body["reason"] + + +def test_runner_auto_terminal_adopts_the_verified_mode_only_when_the_records_pass_the_bar( + tmp_path, ready_workspace): + """Existence is not satisfaction, and the check runs BEFORE the event is appended. + + Both halves live in one test because the acceptance count is fixed: they are the two + sides of a single decision — the runner evaluates the shared predicate against every + candidate record, then chooses the mode. Evaluating it afterwards (what `emit` + already did) would leave `terminal_written` durable and then raise, which is exactly + the committed-then-refused seam this slice removes. + """ + dispatch_once(ready_workspace) + dispatch_once(ready_workspace) + terminal = json.loads((ready_workspace / ".loop" / "terminal_state.json") + .read_text(encoding="utf-8")) + assert terminal["completion_policy"] == {"mode": VERIFIED_EVIDENCE_MODE} + assert terminal["evidence"] == [".loop/evidence/evidence-iter1.json"] + + moved = _ready(tmp_path / "moved", [_task("T-1")]) + dispatch_once(moved) + tasks_path = moved / "TASKS.json" # move the goalpost the record cites + data = json.loads(tasks_path.read_text(encoding="utf-8")) + data["tasks"][0]["verify"] = "true" + tasks_path.write_text(json.dumps(data), encoding="utf-8") + result = dispatch_once(moved) # must not raise, and must not lie + assert result["action"] == "terminal_written" + written = json.loads((moved / ".loop" / "terminal_state.json").read_text(encoding="utf-8")) + assert written["completion_policy"] == {"mode": "all_required"} + assert written["evidence"] == ["RUNLOG.md"] + assert ".loop/evidence/evidence-iter1.json" in written["reason"] + assert "goalpost" in written["reason"] + # The durable event says the same thing the file does: one decision, taken once. + event = SQLiteEventStore(moved / ".loop" / "events.db").read("run-1")[-1] + assert event["type"] == "terminal_written" + assert event["payload"]["completion_policy"] == {"mode": "all_required"} + assert event["payload"]["reason"] == written["reason"] + + +def test_runner_auto_terminal_keeps_all_required_when_a_task_has_no_record(predone_workspace): + dispatch_once(predone_workspace) + terminal = json.loads((predone_workspace / ".loop" / "terminal_state.json") + .read_text(encoding="utf-8")) + assert terminal["completion_policy"] == {"mode": "all_required"} + assert terminal["evidence"] == ["RUNLOG.md"] + assert "no evidence record" in terminal["reason"] + + +@pytest.mark.parametrize("mode", _MODES) +def test_terminal_schema_accepts_both_modes(ready_workspace, mode): + """terminal@1 admits the new mode in BOTH validation modes, end to end. + + Driven through the runner's own auto-terminal rather than a bare emit.terminate: + a hand-terminated dispatched workspace is doctor-dirty for unrelated reasons (a + terminal file with no terminal_written event is a desynced terminal window), so + `ok is True` there would prove nothing about the schema. + """ + dispatch_once(ready_workspace) + dispatch_once(ready_workspace) + report = doctor_report(ready_workspace, mode=mode) + assert report["ok"] is True, report["issues"] diff --git a/scripts/test_loop_simulate_cli.py b/scripts/test_loop_simulate_cli.py index 31d0251..49052f3 100644 --- a/scripts/test_loop_simulate_cli.py +++ b/scripts/test_loop_simulate_cli.py @@ -83,7 +83,7 @@ def test_simulate_on_execute_task_with_undeclared_or_unparseable_verify_command_ def test_simulate_on_execute_task_with_all_tasks_done_predicts_would_write_terminal_with_exact_payload_preview(tmp_path): w, _ = _ws(tmp_path, [_task(status="done")]); r = simulate_run(w)["would"] - assert r["action"] == "would_write_terminal" and r["predicted_terminal"] == {"state": "Succeeded", "criteria_met": {"T-1": True}, "evidence": ["RUNLOG.md"], "false_completion": False, "completion_policy": {"mode": "all_required"}, "iteration_id": 4} + assert r["action"] == "would_write_terminal" and r["predicted_terminal"] == {"state": "Succeeded", "criteria_met": {"T-1": True}, "evidence": ["RUNLOG.md"], "false_completion": False, "completion_policy": {"mode": "all_required"}, "iteration_id": 4, "reason": "tasks with no evidence record: T-1"} def test_simulate_on_execute_task_with_unsatisfiable_dependency_predicts_would_block_with_null_refusal_reason(tmp_path): From 94cc226d0135f2204f9d2581ec0bd8134f59206b Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 16:28:27 -0400 Subject: [PATCH 08/22] fix(completion): make the write-time evidence bar a superset of the pure one The workspace-bearing layers checked the record's internal uri for containment but never the cited entry path itself, so a store-less contract could write a Succeeded terminal citing a record outside the workspace -- accepted by emit.terminate, refused by the reducer replaying that same terminal. Reuse the pure layers' own shape predicate so the four layers agree on what Succeeded means. --- loop/contract.py | 6 +++++ scripts/test_completion_verified_evidence.py | 24 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/loop/contract.py b/loop/contract.py index fbda791..a36e496 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -12,6 +12,7 @@ VERIFIED_EVIDENCE_MODE, CompletionPolicyError, criteria_satisfy_completion, + evidence_entry_is_record_shaped, normalize_completion_policy, policy_requires_verified_evidence, unmet_required_criteria, @@ -815,6 +816,11 @@ def _strict_evidence_failure(entry: object, paths: LoopPaths, if not isinstance(entry, str) or not entry.strip(): return "is not a workspace-relative evidence record path" + if not evidence_entry_is_record_shaped(entry): + # The pure layers reject this shape outright, so accepting it here would let a + # terminal through the writer that its own replay would refuse. + return ("is not verified evidence: not a workspace-relative " + ".loop/evidence/*.json record path") try: blob = (paths.workspace / entry).read_bytes() except (OSError, ValueError) as exc: diff --git a/scripts/test_completion_verified_evidence.py b/scripts/test_completion_verified_evidence.py index 1b7fbb2..f0e732b 100644 --- a/scripts/test_completion_verified_evidence.py +++ b/scripts/test_completion_verified_evidence.py @@ -15,8 +15,8 @@ import pytest from loop import emit, integrations -from loop.completion import (VERIFIED_EVIDENCE_MODE, normalize_completion_policy, - policy_requires_verified_evidence) +from loop.completion import (VERIFIED_EVIDENCE_MODE, evidence_entry_is_record_shaped, + normalize_completion_policy, policy_requires_verified_evidence) from loop.contract import doctor_report from loop.events import SQLiteEventStore from loop.reducer import EventReplayError, reduce_events @@ -209,6 +209,26 @@ def test_doctor_reports_an_unbound_record_cited_by_a_strict_terminal(dispatched_ assert "unverified_evidence_terminal" in _codes(doctor_report(dispatched_workspace)) +def test_the_write_layers_reject_an_entry_the_pure_layers_would_reject(tmp_path): + """The workspace-bearing bar must be a SUPERSET of the pure one. + + A store-less contract skips chain-boundness by design, so without an entry-shape + check a Succeeded terminal could cite a record living outside the workspace: written + happily by ``emit.terminate``, then refused by the reducer replaying that same + terminal. Layers that disagree about what Succeeded means are the defect. + """ + workspace = tmp_path / "storeless" + emit.open_contract(workspace) + outside = tmp_path / "elsewhere" / "rec.json" + outside.parent.mkdir(parents=True, exist_ok=True) + outside.write_text("{}", encoding="utf-8") + entry = "../elsewhere/rec.json" + assert evidence_entry_is_record_shaped(entry) is False + with pytest.raises(emit.EmitError, match="record path"): + emit.terminate(workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[entry], completion_policy=VERIFIED_EVIDENCE_MODE) + + def _corrupt_store(workspace): """Make the event store genuinely unreadable, at the bytes. From 7b3e5d8d3fb07a49e2cc3c92c12f408e71433f2c Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 15:36:31 -0400 Subject: [PATCH 09/22] feat(metrics): an injected-callable verdict is not gate evidence --- scripts/metrics.py | 27 ++++++++- scripts/test_metrics_verifier_source.py | 73 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 scripts/test_metrics_verifier_source.py diff --git a/scripts/metrics.py b/scripts/metrics.py index 046d6bf..84dd3fb 100644 --- a/scripts/metrics.py +++ b/scripts/metrics.py @@ -28,6 +28,13 @@ red gate, and a same-iteration green never excuses its own task's red — within-iteration chronology is unknowable, so it fails closed. +A verify bundle whose `verifier.source` is `injected_callable` is a caller's +self-report, not an independently-executed gate, so it is excluded from every +metric before FCR and RP are derived; the excluded filenames are listed under +`provenance.injected_verifier_bundles` so the exclusion is visible rather than a +silent FCR shift. A bundle carrying no `verifier` block at all has an unknown +source and still counts — grandfathering by absence, not by guess. + The `### Outcome` token contract: a claim counts toward the FCR denominator only when its outcome token is a recognized SUCCESS token — completion-class (`task_passed`, `terminal`, `succeeded`) or progress-class (`advanced`); @@ -107,6 +114,11 @@ # invocation runs. evidence_backed via a verify script requires one to exist. _GATE_SCRIPTS = ("holdout_gate.py", "anticheat_scan.py", "anti_cheat.py") +# A verify bundle whose `verifier.source` is this ran a caller-supplied callable +# (`loop/verifier.py:111`), so its verdict is a self-report rather than the output +# of an independently-resolved command. It is never gate evidence. +_INJECTED_VERIFIER_SOURCE = "injected_callable" + _ITER_HEADER_RE = re.compile(r"(?m)^##\s+Iteration\s+(\S+)") # "outcome" declaration followed (within a little markup/whitespace) by its token. _OUTCOME_RE = re.compile(r"outcome[^A-Za-z0-9]{0,40}?([A-Za-z][A-Za-z_]{1,})", re.IGNORECASE | re.DOTALL) @@ -239,6 +251,8 @@ def _load_verify_bundles(loop_dir: Path) -> list[dict]: green = outcome == "PASS" or data.get("passed") is True it = data.get("iteration_id", data.get("iteration")) task = data.get("task") + verifier = data.get("verifier") + source = verifier.get("source") if isinstance(verifier, dict) else None bundles.append( { "path": path, @@ -247,6 +261,7 @@ def _load_verify_bundles(loop_dir: Path) -> list[dict]: "iter": _norm_iter(it) if it is not None else None, "task": str(task) if task is not None else None, "score": _num(data.get("score")), + "source": source, } ) return bundles @@ -455,7 +470,16 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict terminal = _read_json_object(paths.terminal) blocks = _runlog_blocks(runlog_text) - bundles = _load_verify_bundles(loop_dir_path) + # An injected-callable verdict is the caller's own self-report, not an + # independently-executed gate (`reference/repo-os-contract.md` §17), so it is + # partitioned out before any metric consumes it. Only an EXPLICIT + # `injected_callable` source is excluded: a bundle with no `verifier` block has + # an unknown source and keeps counting — grandfathering by absence, not by + # guess. The excluded names are reported under provenance so the exclusion is + # visible rather than a silent FCR shift. + all_bundles = _load_verify_bundles(loop_dir_path) + injected = sorted(b["name"] for b in all_bundles if b["source"] == _INJECTED_VERIFIER_SOURCE) + bundles = [b for b in all_bundles if b["source"] != _INJECTED_VERIFIER_SOURCE] verify_by_iter, unmatched_verify = _assign_bundles_to_iters(bundles, blocks) # Success claims: RUNLOG success-outcome iterations, plus the terminal claim. @@ -600,6 +624,7 @@ def _claim_is_clean(iid: str) -> bool: "holdout_source": holdout_source, "holdout_verdicts": holdout_verdicts, "unmatched_verify": unmatched_verify, + "injected_verifier_bundles": injected, }, } diff --git a/scripts/test_metrics_verifier_source.py b/scripts/test_metrics_verifier_source.py new file mode 100644 index 0000000..5fafb00 --- /dev/null +++ b/scripts/test_metrics_verifier_source.py @@ -0,0 +1,73 @@ +"""An injected-callable verdict is a caller's self-report, not gate evidence.""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from metrics import compute_metrics # noqa: E402 + +_ROOT = Path(__file__).resolve().parent.parent + + +def _bundle(source=None, *, green=True, task="T1", iteration_id=1): + body = {"iteration_id": iteration_id, "task": task, + "outcome": "PASS" if green else "FAIL", "passed": green, "score": 1.0 if green else 0.0} + if source is not None: + body["verifier"] = {"by": "loop.run", "source": source} + return body + + +def _ws(tmp_path, bundles): + target = tmp_path / "workspace" + (target / ".loop" / "artifacts").mkdir(parents=True, exist_ok=True) + (target / "RUNLOG.md").write_text( + "# RUNLOG\n\n## Iteration 1 — 2026-07-25\n\n### Outcome\n\n`task_passed`\n", + encoding="utf-8") + (target / ".loop" / "state.json").write_text(json.dumps( + {"schema": "loop-engineer/state@1", "state": "execute-task", "iteration_id": 1, + "terminal_state": None}), encoding="utf-8") + for name, body in bundles: + (target / ".loop" / "artifacts" / name).write_text(json.dumps(body), encoding="utf-8") + return target + + +def test_a_declared_command_bundle_is_gate_evidence(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("declared_command"))]) + assert compute_metrics(target)["false_completion_rate"] == 0.0 + + +def test_an_injected_callable_bundle_is_not_gate_evidence(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable"))]) + assert compute_metrics(target)["provenance"]["injected_verifier_bundles"] == ["verify-iter1.json"] + + +def test_an_injected_only_claim_is_a_false_completion(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable"))]) + assert compute_metrics(target)["false_completion_rate"] == 1.0 + + +def test_a_legacy_bundle_without_a_verifier_block_still_counts(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle(None))]) + metrics = compute_metrics(target) + assert metrics["false_completion_rate"] == 0.0 + assert metrics["provenance"]["injected_verifier_bundles"] == [] + + +def test_excluded_bundles_are_named_under_provenance(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable")), + ("verify-iter2.json", _bundle("declared_command", iteration_id=1))]) + assert compute_metrics(target)["provenance"]["injected_verifier_bundles"] == ["verify-iter1.json"] + + +def test_injected_bundles_do_not_anchor_a_repair(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable", green=False)), + ("verify-iter2.json", _bundle("injected_callable"))]) + assert compute_metrics(target)["repair_productivity"] is None + + +def test_the_shipped_examples_keep_their_published_metrics(): + metrics = compute_metrics(_ROOT / "examples" / "flaky-test-triage") + assert metrics["false_completion_rate"] == 0.0 + assert metrics["repair_productivity"] == 1.0 From bffeb5126aa6b1bb3f3e8929acdb949da3aa165b Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 15:46:44 -0400 Subject: [PATCH 10/22] test(metrics): make the repair-anchoring pin discriminate --- scripts/test_metrics_verifier_source.py | 35 ++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/scripts/test_metrics_verifier_source.py b/scripts/test_metrics_verifier_source.py index 5fafb00..55f4af7 100644 --- a/scripts/test_metrics_verifier_source.py +++ b/scripts/test_metrics_verifier_source.py @@ -19,7 +19,17 @@ def _bundle(source=None, *, green=True, task="T1", iteration_id=1): return body -def _ws(tmp_path, bundles): +def _repair(*, before, after, iteration_id="iter-001"): + return {"schema": "loop-engineer/repair@1", "iteration_id": iteration_id, "attempt": 1, + "failure_mode": "deterministic-fail", "hypothesis": "h", "repair_action": "a", + "verification_before": {"verify_full": "FAIL", "metric": "score", + "failing": ["x"], "score": before}, + "verification_after": {"verify_full": "PASS", "metric": "score", + "failing": [], "score": after}, + "remaining_delta": "none", "productive": True} + + +def _ws(tmp_path, bundles, repairs=()): target = tmp_path / "workspace" (target / ".loop" / "artifacts").mkdir(parents=True, exist_ok=True) (target / "RUNLOG.md").write_text( @@ -30,6 +40,10 @@ def _ws(tmp_path, bundles): "terminal_state": None}), encoding="utf-8") for name, body in bundles: (target / ".loop" / "artifacts" / name).write_text(json.dumps(body), encoding="utf-8") + if repairs: + (target / ".loop" / "repair").mkdir(parents=True, exist_ok=True) + for name, body in repairs: + (target / ".loop" / "repair" / name).write_text(json.dumps(body), encoding="utf-8") return target @@ -62,9 +76,22 @@ def test_excluded_bundles_are_named_under_provenance(tmp_path): def test_injected_bundles_do_not_anchor_a_repair(tmp_path): - target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable", green=False)), - ("verify-iter2.json", _bundle("injected_callable"))]) - assert compute_metrics(target)["repair_productivity"] is None + """An entirely self-reported red-to-green pair must not corroborate a repair. + + The repair record is load-bearing: without one on disk, ``repair_productivity`` + is ``None`` for the unrelated reason that nothing was validated, and the test + passes against unchanged code. With one, the pair anchors before this change + and cannot after it. + + It deliberately does not claim more than it proves. An unanchored record still + counts toward plain-mode ``repair_productivity``; only ``--baseline`` refuses. + """ + target = _ws(tmp_path, + [("verify-iter1.json", _bundle("injected_callable", green=False)), + ("verify-iter2.json", _bundle("injected_callable"))], + repairs=[("iter-001.json", _repair(before=0.0, after=1.0))]) + assert compute_metrics(target)["provenance"]["unanchored_records"] == [ + ".loop/repair/iter-001.json"] def test_the_shipped_examples_keep_their_published_metrics(): From eb7b09bcf23435777594d9f77caa1a7d2148beec Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 16:56:01 -0400 Subject: [PATCH 11/22] test(adversarial): flip the four closed pins and pin what binding still misses Flips four Slice-2 pins in place (renamed with bodies, residual halves kept, each docstring naming the release that changed the behavior) and adds scripts/test_adversarial_evidence_binding.py - 14 pins stating exactly what chain-bound evidence catches and what it provably does not. Pins 13 and 14 hold decision 14 two residuals. Pin 13 corrects the plan claim that doctor missing_event_store tripwire covers the delete-the-store route: measured on a real dispatch it does not, because that tripwire is gated on leftover -wal/-shm sidecars and sidecar-free reads leave none, so deleting the store makes plain doctor report ok with zero issues. The pin asserts that measured silence and uses --expect-chain-head as its positive control. Also discharges carry-forward 11: the workspace-root symlink-loop variant of the argv[0] pin, green on 3.12 (pathlib RuntimeError) and 3.13 (OSError). --- scripts/test_adversarial_evidence_binding.py | 538 ++++++++++++++++++ scripts/test_adversarial_verifier_identity.py | 116 +++- scripts/test_verifier_identity.py | 14 + 3 files changed, 648 insertions(+), 20 deletions(-) create mode 100644 scripts/test_adversarial_evidence_binding.py diff --git a/scripts/test_adversarial_evidence_binding.py b/scripts/test_adversarial_evidence_binding.py new file mode 100644 index 0000000..c925f8a --- /dev/null +++ b/scripts/test_adversarial_evidence_binding.py @@ -0,0 +1,538 @@ +"""What chain-bound evidence catches, and what it provably does not. + +The claim this file defends is "evidence whose bytes are bound into an ANCHORABLE +chain", never "tamper-proof evidence". Binding inherits Slice 1's trust model +exactly: a worker who can rewrite .loop/ can rewrite the store (repo-os-contract +#16, Integrity boundary), so without an external anchor a full rewrite verifies. + +Every ``*_pinned`` test states an honest limitation and carries the positive control +that bounds it: a pin whose detector was never shown to fire certifies a hole as +closed. The rest prove a real detection. +""" +from __future__ import annotations + +import hashlib +import json +import shutil +import sqlite3 +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from metrics import compute_metrics # noqa: E402 + +from chain_fixtures import drop_triggers, restore_triggers # noqa: E402 +from loop import emit # noqa: E402 +from loop.chain import compute_event_hash # noqa: E402 +from loop.completion import VERIFIED_EVIDENCE_MODE # noqa: E402 +from loop.contract import doctor_report # noqa: E402 +from loop.events import SQLiteEventStore # noqa: E402 +from loop.evidence import artifact_object_path # noqa: E402 +from loop.runner import dispatch_once # noqa: E402 +from loop.runtime import bound_artifact_digests # noqa: E402 +from loop.verifier import (executed_verifier_identity, # noqa: E402 + verification_policy_digest, verifier_code_digest) + +_EVENT_SCHEMA_ID = "loop-engineer/event@1" +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RUN_ID = "run-1" +_FORGED = '{"outcome": "PASS", "passed": true, "forged": true}\n' + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _task(task_id="T-1", **overrides): + base = {"id": task_id, "title": task_id, "status": "pending", "criterion_ref": task_id, + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None} + base.update(overrides) + return base + + +def _write_tasks(workspace, tasks): + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": list(tasks)}), encoding="utf-8") + + +def _contract(tmp_path, name="workspace", tasks=(), *, store=True): + """A contract, optionally with an event store ramped to execute-task. + + The ramp records the FSM walk, not work: its events carry iteration_id 0, so the + first dispatch is iteration 1 and every artifact path asserted below is literal. + Without a store the contract is the supported writer-API path — the one decision + 14 names as unable to chain-bind anything. + """ + workspace = tmp_path / name + emit.open_contract(workspace) + if tasks: + _write_tasks(workspace, tasks) + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + if store: + events = SQLiteEventStore(_store_path(workspace)) + events.append(_RUN_ID, "contract_opened", {"workspace": name}, actor="test") + for state in _RAMP: + events.append(_RUN_ID, "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + # The ramp is written straight to the store, so state.json still says intake. + # Doctor reconciles the two, so the fixture must land where the ramp landed. + state_path = workspace / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["state"] = _RAMP[-1] + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + return workspace + + +def _dispatched(tmp_path, name="workspace"): + workspace = _contract(tmp_path, name, [_task()]) + dispatch_once(workspace) + return workspace + + +def _store_path(workspace): + return workspace / ".loop" / "events.db" + + +def _bundle(workspace): + return workspace / ".loop" / "artifacts" / "verify-iter1.json" + + +def _record(workspace): + return workspace / ".loop" / "evidence" / "evidence-iter1.json" + + +def _head(workspace): + return doctor_report(workspace)["event_store"]["chain"]["head"]["event_hash"] + + +def _record_at(conn, sequence, prev_event_hash): + """Rebuild one row into the record dict read_event_rows projects (hash preimage shape).""" + row = conn.execute( + "SELECT run_id, sequence, event_id, type, actor, causation_id, correlation_id, ts, " + "payload, artifact_hashes FROM events WHERE sequence = ?", (sequence,)).fetchone() + return {"schema": _EVENT_SCHEMA_ID, "run_id": row[0], "sequence": row[1], "event_id": row[2], + "type": row[3], "actor": row[4], "causation_id": row[5], "correlation_id": row[6], + "ts": row[7], "payload": json.loads(row[8]), "artifact_hashes": json.loads(row[9]), + "prev_event_hash": prev_event_hash} + + +def _rechain(workspace, artifact_hashes): + """Replace the bound digests on the one binding event and re-chain from genesis.""" + store_path = _store_path(workspace) + drop_triggers(store_path) + conn = sqlite3.connect(str(store_path)) + try: + conn.execute("UPDATE events SET artifact_hashes = ? WHERE artifact_hashes != '[]'", + (json.dumps(artifact_hashes),)) + prev = None + for (sequence,) in conn.execute( + "SELECT sequence FROM events ORDER BY sequence ASC").fetchall(): + digest = compute_event_hash(_record_at(conn, sequence, prev)) + conn.execute("UPDATE events SET prev_event_hash = ?, event_hash = ? WHERE sequence = ?", + (prev, digest, sequence)) + prev = digest + conn.commit() + finally: + conn.close() + restore_triggers(store_path) + + +def _rewrite_everything(workspace, *, text=_FORGED): + """The competent adversary: new bundle bytes, a matching record, a matching object, + the bound row's artifact_hashes replaced, and the chain recomputed from genesis.""" + bundle, record_path = _bundle(workspace), _record(workspace) + record = json.loads(record_path.read_text(encoding="utf-8")) + artifact_object_path(workspace, record["sha256"]).unlink() # burn the recovery copy + bundle.write_text(text, encoding="utf-8") + new_sha = hashlib.sha256(text.encode("utf-8")).hexdigest() + record["sha256"] = new_sha + record_text = json.dumps(record, indent=2, sort_keys=True) + "\n" + record_path.write_text(record_text, encoding="utf-8") + obj = artifact_object_path(workspace, new_sha) + obj.parent.mkdir(parents=True, exist_ok=True) + obj.write_text(text, encoding="utf-8") + _rechain(workspace, [ + {"path": bundle.relative_to(workspace).as_posix(), "sha256": new_sha}, + {"path": record_path.relative_to(workspace).as_posix(), + "sha256": hashlib.sha256(record_text.encode("utf-8")).hexdigest()}, + {"path": obj.relative_to(workspace).as_posix(), "sha256": new_sha}, + ]) + return new_sha + + +def _unbind(workspace): + """Replay the stream into a fresh store with nothing bound — a pre-release run. + + The append-only triggers make retroactive UNbinding as impossible as retroactive + binding, so the only way a pre-binding contract can exist is the way it was + originally written: every event re-appended verbatim except its artifact_hashes. + """ + events = SQLiteEventStore(_store_path(workspace)).read(_RUN_ID) + for suffix in ("", "-wal", "-shm"): + _store_path(workspace).with_name("events.db" + suffix).unlink(missing_ok=True) + store = SQLiteEventStore(_store_path(workspace)) + for event in events: + store.append(_RUN_ID, event["type"], event["payload"], actor=event["actor"], + event_id=event["event_id"], causation_id=event["causation_id"], + correlation_id=event["correlation_id"], ts=event["ts"], + artifact_hashes=None) + + +def _handwritten_record(workspace, *, task_id="T-1"): + """A self-consistent, never-dispatched record: real bundle, real digest, real policy.""" + bundle = workspace / ".loop" / "artifacts" / "verify-iter9.json" + bundle.parent.mkdir(parents=True, exist_ok=True) + text = '{"outcome": "PASS", "passed": true}' + bundle.write_text(text, encoding="utf-8") + tasks = json.loads((workspace / "TASKS.json").read_text(encoding="utf-8")) + entry = next(t for t in tasks["tasks"] if t["id"] == task_id) + record = { + "schema": "loop-engineer/evidence@1", "id": "hand:9:verify", "kind": "verify-bundle", + "uri": ".loop/artifacts/verify-iter9.json", + "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), + "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", + "produced_by": {"run_id": _RUN_ID, "task_id": task_id, "attempt": 1, + "executor": "worker-a"}, + "verified_by": {"by": "ci", "at": "2026-07-25T00:00:00+00:00", + "command": entry["verify"], "code_digest": None, + "code_digest_basis": "path_lookup", + "policy_digest": verification_policy_digest(entry)}, + } + path = workspace / ".loop" / "evidence" / "evidence-iter9.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return ".loop/evidence/evidence-iter9.json" + + +def _terminal_file(workspace, *, mode, evidence): + (workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "project": workspace.name, "state": "Succeeded", + "criteria_met": {"T-1": True}, "completion_policy": {"mode": mode}, + "evidence": list(evidence), "false_completion": False, + "terminated_at": "2026-07-25T00:00:00+00:00", "reason": "hand-written", + }, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _metrics_workspace(tmp_path, source): + """A minimal metrics-readable loop dir: one green claim backed by one green bundle.""" + target = tmp_path / "metrics-ws" + (target / ".loop" / "artifacts").mkdir(parents=True, exist_ok=True) + (target / "RUNLOG.md").write_text( + "# RUNLOG\n\n## Iteration 1 — 2026-07-25\n\n### Outcome\n\n`task_passed`\n", + encoding="utf-8") + (target / ".loop" / "state.json").write_text(json.dumps( + {"schema": "loop-engineer/state@1", "state": "execute-task", "iteration_id": 1, + "terminal_state": None}), encoding="utf-8") + (target / ".loop" / "artifacts" / "verify-iter1.json").write_text(json.dumps( + {"iteration_id": 1, "task": "T-1", "outcome": "PASS", "passed": True, "score": 1.0, + "verifier": {"by": "loop.run", "source": source}}), encoding="utf-8") + return target + + +# --- pinned honest limitations ------------------------------------------------- + +def test_a_full_rewrite_of_artifacts_and_store_is_not_caught_without_an_anchor_pinned(tmp_path): + """Binding proves the bytes AGREE, never that a dispatch happened. + + A worker who can rewrite .loop/ owns the store too: rewrite the bundle, recompute + the record and the content-addressed object, replace the digests the event bound and + re-chain from genesis, and every internal check agrees again. This is Slice 1's + trust model unchanged (repo-os-contract #16) — which is why the shipped claim is + "bound into an ANCHORABLE chain" and never "tamper-proof". The anchor that does + catch it is the very next test. + """ + workspace = _dispatched(tmp_path) + assert doctor_report(workspace)["ok"] is True # control: clean before + before = bound_artifact_digests(workspace)[".loop/artifacts/verify-iter1.json"] + new_sha = _rewrite_everything(workspace) + # the forge is real, not a no-op: different bytes, and the chain now commits to them + assert new_sha != before + assert bound_artifact_digests(workspace)[".loop/artifacts/verify-iter1.json"] == new_sha + assert json.loads(_bundle(workspace).read_text(encoding="utf-8"))["forged"] is True + report = doctor_report(workspace) + assert report["ok"] is True and report["issues"] == [] + + +def test_a_record_written_outside_a_dispatch_is_never_chain_bound_pinned(tmp_path): + """``emit.write_verify_evidence`` is a WRITER, not a dispatch: it appends no event. + + Nothing binds what it wrote, so a later rewrite of that record is invisible. The + writer-API path is supported deliberately (decision 14's residual (i)), so this + silence ships NAMED rather than closed. Positive control: the identical rewrite in a + dispatched contract is caught, which is the only thing that makes this a boundary + rather than an absence of checking. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + written = emit.write_verify_evidence( + workspace, run_id=_RUN_ID, iteration_id=1, task=_task(), passed=True, + code_identity=executed_verifier_identity(_VERIFY, workspace)) + assert bound_artifact_digests(workspace) is None # no store: nothing can bind + record = json.loads(written["evidence"].read_text(encoding="utf-8")) + record["verified_by"]["by"] = "ci" + written["evidence"].write_text(json.dumps(record), encoding="utf-8") + assert "evidence_chain_mismatch" not in _codes(doctor_report(workspace)) + + bound = _dispatched(tmp_path, "bound") # control: a dispatch DOES bind + data = json.loads(_record(bound).read_text(encoding="utf-8")) + data["verified_by"]["by"] = "ci" + _record(bound).write_text(json.dumps(data), encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(bound)) + + +def test_a_legacy_iteration_event_can_never_be_bound_retroactively_pinned(tmp_path): + """A pre-release event carries ``artifact_hashes: []`` and is silent by construction. + + It cannot be repaired either: the append-only triggers forbid the UPDATE a backfill + would need — the same reason ``loop migrate`` refuses to backfill chain hashes. So a + pre-release iteration whose evidence was deleted stays undetectable at this layer + forever. Positive control: the same deletion over a BOUND event is reported. + """ + workspace = _dispatched(tmp_path) + _unbind(workspace) + assert bound_artifact_digests(workspace) == {} + _record(workspace).unlink() + _bundle(workspace).unlink() + assert "missing_bound_evidence" not in _codes(doctor_report(workspace)) + with pytest.raises((sqlite3.IntegrityError, sqlite3.OperationalError), match="append-only"): + conn = sqlite3.connect(str(_store_path(workspace))) + try: + conn.execute("UPDATE events SET artifact_hashes = '[]' WHERE sequence = 1") + finally: + conn.close() + + bound = _dispatched(tmp_path, "bound") # control: a bound pair IS missed + _record(bound).unlink() + _bundle(bound).unlink() + assert "missing_bound_evidence" in _codes(doctor_report(bound)) + + +def test_doctor_does_not_re_hash_the_verifier_file_pinned(tmp_path): + """``code_digest`` stays in the recorded-not-compared tier (decision 5). + + A verifier file legitimately changes between runs, and a comparison with no declared + baseline would fire on every honest edit. Only what the chain BOUND is re-hashed, and + the verifier script is not a bound artifact — so editing it after the dispatch leaves + doctor clean. Control: the bytes demonstrably moved, and the record still says + otherwise. + """ + workspace = _dispatched(tmp_path) + recorded = json.loads(_record(workspace).read_text(encoding="utf-8"))["verified_by"]["code_digest"] + script = workspace / "scripts" / "verify-fast.sh" + script.write_text("#!/bin/sh\nexit 0\n# tampered\n", encoding="utf-8") + assert verifier_code_digest(_VERIFY, workspace)[0] != recorded # control: bytes moved + assert "scripts/verify-fast.sh" not in bound_artifact_digests(workspace) + assert doctor_report(workspace)["ok"] is True + + +def test_an_older_record_for_a_moved_goalpost_is_not_compared_pinned(tmp_path): + """Decision 5's cost, stated plainly: only the LATEST record per task is compared. + + A superseded record therefore keeps a goalpost nobody checks. The alternative — + comparing every record — turns each honest re-verification into a permanent doctor + failure, because the older record legitimately describes the goalpost that was + current when it was written. Positive control: move the goalpost the LATEST record + cites and the comparison fires. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + identity = executed_verifier_identity(_VERIFY, workspace) + emit.write_verify_evidence(workspace, run_id=_RUN_ID, iteration_id=1, task=_task(), + passed=False, code_identity=identity) + emit.write_verify_evidence(workspace, run_id=_RUN_ID, iteration_id=2, + task=_task(verify="true"), passed=True, code_identity=identity) + _write_tasks(workspace, [_task(verify="true")]) # the LATEST record's goalpost + stale = json.loads(_record(workspace).read_text(encoding="utf-8"))["verified_by"]["policy_digest"] + assert stale != verification_policy_digest(_task(verify="true")) # iter1 IS stale + assert "policy_digest_mismatch" not in _codes(doctor_report(workspace)) + + _write_tasks(workspace, [_task(verify="false")]) # control: move the LATEST goalpost + assert "policy_digest_mismatch" in _codes(doctor_report(workspace)) + + +def test_deleting_every_artifact_and_the_store_leaves_a_clean_doctor_pinned(tmp_path): + """Delete the whole run and the contract is one that never ran. + + Artifacts, records, store AND sidecars gone: nothing inside the tree remembers the + dispatch, so doctor is clean and correct to be (repo-os-contract §22 — the + ``missing_event_store`` tripwire fires only on the sloppier deletion that leaves + -wal/-shm behind). Detection needs something OUTSIDE the tree; the anchored control + at the end of this test is that something. + """ + workspace = _dispatched(tmp_path) + anchor = _head(workspace) + shutil.rmtree(workspace / ".loop" / "artifacts") + shutil.rmtree(workspace / ".loop" / "evidence") + for suffix in ("", "-wal", "-shm"): + _store_path(workspace).with_name("events.db" + suffix).unlink(missing_ok=True) + report = doctor_report(workspace) + assert report["ok"] is True and report["issues"] == [] + assert "chain_anchor_mismatch" in _codes(doctor_report(workspace, expect_chain_head=anchor)) + + +def test_the_verified_evidence_mode_is_opt_in_and_not_retroactive_pinned(tmp_path): + """A Succeeded terminal written under ``all_required`` keeps the OLD bar forever. + + Non-empty path strings remain sufficient there, which is why both shipped examples + and every pre-release run survive this slice unchanged (Global Constraints: the + tightening is opt-in, never retroactive). The positive control is exactly one field: + the byte-identical terminal under the strict mode is reported. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + _terminal_file(workspace, mode="all_required", evidence=["RUNLOG.md"]) + assert "unverified_evidence_terminal" not in _codes(doctor_report(workspace)) + _terminal_file(workspace, mode=VERIFIED_EVIDENCE_MODE, evidence=["RUNLOG.md"]) + assert "unverified_evidence_terminal" in _codes(doctor_report(workspace)) + + +def test_a_hand_written_record_pointing_at_a_real_file_still_passes_pinned(tmp_path): + """Hash verification proves the POINTER, never the PROVENANCE. + + A record hand-written to describe a file that exists is, to doctor, indistinguishable + from one a dispatch produced: same shape, same true digest, same live goalpost. What + this release added is that the pointer must be TRUE — the control below corrupts the + pointed-at bytes and the new check fires. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + _handwritten_record(workspace) + assert doctor_report(workspace)["ok"] is True + (workspace / ".loop" / "artifacts" / "verify-iter9.json").write_text("{}", encoding="utf-8") + assert "hash_mismatch" in _codes(doctor_report(workspace)) # control: it fires + + +def test_criterion_text_is_still_unbound_pinned(tmp_path): + """What a criterion MEANS is prose, and prose is bound by nothing. + + ``policy_digest`` covers id / verify / criterion_ref / depends_on — the POINTER to a + criterion, not its wording — so rewriting SPEC.md's acceptance text moves no digest + and fires nothing. Binding evidence does not bind intent. Control: the bundle IS a + bound artifact and SPEC.md is not, so the difference is structural, not incidental. + """ + workspace = _dispatched(tmp_path) + recorded = json.loads(_record(workspace).read_text(encoding="utf-8"))["verified_by"]["policy_digest"] + live = json.loads((workspace / "TASKS.json").read_text(encoding="utf-8"))["tasks"][0] + spec = workspace / "SPEC.md" + spec.write_text(spec.read_text(encoding="utf-8").replace( + "REPLACE: first success criterion", "anything at all now counts as success"), + encoding="utf-8") + assert "anything at all" in spec.read_text(encoding="utf-8") # the goal moved + assert verification_policy_digest(live) == recorded # the digest did not + bound = bound_artifact_digests(workspace) + assert "SPEC.md" not in bound and ".loop/artifacts/verify-iter1.json" in bound + assert doctor_report(workspace)["ok"] is True + + +def test_metrics_still_counts_a_hand_written_declared_command_bundle_pinned(tmp_path): + """``verifier.source`` is a string the WRITER declares, not a fact metrics can check. + + Hand-writing ``declared_command`` over ``injected_callable`` restores the bundle's + gate-evidence status and moves FCR back to 0.0. The exclusion is an honesty aid for + honest writers, never a control against a dishonest one. Control: the same bundle, + one string different, scores the opposite way. + """ + loop_dir = _metrics_workspace(tmp_path, "injected_callable") + assert compute_metrics(loop_dir)["false_completion_rate"] == 1.0 # control: excluded + bundle = loop_dir / ".loop" / "artifacts" / "verify-iter1.json" + body = json.loads(bundle.read_text(encoding="utf-8")) + body["verifier"]["source"] = "declared_command" + bundle.write_text(json.dumps(body), encoding="utf-8") + metrics = compute_metrics(loop_dir) + assert metrics["false_completion_rate"] == 0.0 + assert metrics["provenance"]["injected_verifier_bundles"] == [] + + +def test_the_strict_mode_accepts_a_hand_written_record_in_a_store_less_contract_pinned(tmp_path): + """Decision 14's residual (i), stated as MEASURED rather than as hoped. + + Where no event store exists there is nothing to bind against, so the strict mode + degrades to hash-verification plus goalpost agreement — and a hand-written record + satisfies it with zero dispatch and zero events. + + The plan claimed doctor's ``missing_event_store`` tripwire is what catches deleting a + store out from under a real run. Measured here, it is NOT: that tripwire is gated on + leftover -wal/-shm sidecars, and sidecar-free reads (v0.10.0) leave none, so deleting + the store makes plain doctor CLEANER — ok, zero issues — rather than louder. The real + compensating control is the EXTERNAL ANCHOR, asserted below; a pin that named the + tripwire instead would be a false claim shipped inside an honesty test. + """ + storeless = _contract(tmp_path, "storeless", [_task()], store=False) + entry = _handwritten_record(storeless) + path = emit.terminate(storeless, state="Succeeded", criteria_met={"T-1": True}, + evidence=[entry], completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} # ACCEPTED — residual + + bound = _dispatched(tmp_path, "bound") # control: a store refuses it + with pytest.raises(emit.EmitError, match="not bound"): + emit.terminate(bound, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_handwritten_record(bound)], + completion_policy=VERIFIED_EVIDENCE_MODE) + assert "chain_anchor_mismatch" not in _codes( + doctor_report(bound, expect_chain_head=_head(bound))) # anchor is quiet when intact + + deleted = _dispatched(tmp_path, "deleted") # the delete-the-store route + anchor = _head(deleted) + for suffix in ("", "-wal", "-shm"): + _store_path(deleted).with_name("events.db" + suffix).unlink(missing_ok=True) + silent = doctor_report(deleted) + assert silent["ok"] is True and silent["issues"] == [] # MEASURED: deleting it is silent + assert bound_artifact_digests(deleted) is None + emit.terminate(deleted, state="Succeeded", criteria_met={"T-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) # ACCEPTED — residual + assert "chain_anchor_mismatch" in _codes( + doctor_report(deleted, expect_chain_head=anchor)) # the anchor catches it + + +def test_a_full_rewrite_satisfies_the_strict_mode_without_an_anchor_pinned(tmp_path): + """The headline residual restated at the COMPLETION layer. + + Rewrite the bundle, the record, the object and the digests the event bound, re-chain + the tail, and ``terminate(..., all_required_verified_evidence)`` succeeds: the strict + mode is exactly as strong as the chain it reads, and the chain is as strong as its + anchor. Positive control: the same tree, checked against the pre-rewrite head, reports + ``chain_anchor_mismatch``. This pin is why the shipped claim stays "bound into an + anchorable chain" and why "tamper-proof evidence" is forbidden. + """ + workspace = _dispatched(tmp_path) + anchor = _head(workspace) + _rewrite_everything(workspace) + path = emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} + assert json.loads(_bundle(workspace).read_text(encoding="utf-8"))["forged"] is True + assert "chain_anchor_mismatch" in _codes( + doctor_report(workspace, expect_chain_head=anchor)) # control: the anchor + + +# --- real detections ----------------------------------------------------------- + +def test_an_anchored_head_catches_the_full_rewrite(tmp_path): + """The positive control for the first pin: an anchor recorded outside the tree.""" + workspace = _dispatched(tmp_path) + anchor = _head(workspace) + assert "chain_anchor_mismatch" not in _codes( + doctor_report(workspace, expect_chain_head=anchor)) + _rewrite_everything(workspace) + assert "chain_anchor_mismatch" in _codes( + doctor_report(workspace, expect_chain_head=anchor)) + + +def test_deleting_the_object_alone_is_reported(tmp_path): + """The content-addressed object is a BOUND artifact, not a convenience copy. + + Deleting the recovery source an ``evidence_chain_mismatch`` message points at is + itself detectable, so the recovery path cannot be quietly removed first. + """ + workspace = _dispatched(tmp_path) + digest = json.loads(_record(workspace).read_text(encoding="utf-8"))["sha256"] + obj = artifact_object_path(workspace, digest) + assert obj.is_file() + obj.unlink() + assert "missing_bound_evidence" in _codes(doctor_report(workspace)) diff --git a/scripts/test_adversarial_verifier_identity.py b/scripts/test_adversarial_verifier_identity.py index 03780df..f52b62d 100644 --- a/scripts/test_adversarial_verifier_identity.py +++ b/scripts/test_adversarial_verifier_identity.py @@ -9,12 +9,18 @@ import hashlib import json +import pytest + from loop import emit from loop.contract import doctor_report +from loop.events import SQLiteEventStore from loop.evidence import verify_evidence +from loop.runner import dispatch_once from loop.verifier import (executed_verifier_identity, injected_verifier_identity, verification_policy_digest, verifier_code_digest) +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + def _task(**overrides): base = {"id": "T-1", "title": "t", "status": "pending", "criterion_ref": "C-1", @@ -32,6 +38,37 @@ def _ws(tmp_path): return workspace +@pytest.fixture +def bound_workspace(tmp_path): + """One real dispatch behind an event store: iteration 1's artifacts ARE chain-bound. + + The store's ramp records the FSM walk to execute-task, not work — its events carry + iteration_id 0 — so the first dispatch is iteration 1 and both paths are literal. + """ + workspace = tmp_path / "bound" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [_task(criterion_ref="T-1", title="T-1")]}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "bound"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + state_path = workspace / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["state"] = _RAMP[-1] + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + dispatch_once(workspace) + return {"workspace": workspace, + "bundle": workspace / ".loop" / "artifacts" / "verify-iter1.json", + "record": workspace / ".loop" / "evidence" / "evidence-iter1.json"} + + def _write(workspace, *, iteration_id=1, task=None, passed=True, **kwargs): task = task or _task() kwargs.setdefault("code_identity", @@ -64,10 +101,15 @@ def test_unattributed_executor_never_trips_the_finding_pinned(tmp_path): assert "self_verified_evidence" not in _codes(doctor_report(workspace)) -def test_deleting_both_bundle_and_record_leaves_doctor_clean_pinned(tmp_path): - """Deleting the RECORD alone is caught (missing_evidence_record). Deleting the - PAIR is not: nothing outside the event log remembers the dispatch happened. - Slice 3 closes this by making Succeeded require hash-verified evidence. +def test_deleting_both_bundle_and_record_is_caught_only_when_the_chain_binds_them(tmp_path, + bound_workspace): + """Was `test_deleting_both_bundle_and_record_leaves_doctor_clean_pinned`; the + detection half arrived with chain-bound evidence in the evidence-wiring release. + + The RESIDUAL is unchanged and kept: written through the writer API with no event + store behind it, nothing outside the tree remembers the dispatch happened, so + deleting the pair is still invisible. Where a dispatch bound them, the committed + event names artifacts that are gone — `missing_bound_evidence`. """ workspace = _ws(tmp_path) written = _write(workspace, executor="w", verifier_identity="w") @@ -75,13 +117,21 @@ def test_deleting_both_bundle_and_record_leaves_doctor_clean_pinned(tmp_path): written["evidence"].unlink() assert "missing_evidence_record" in _codes(doctor_report(workspace)) # residue IS caught written["bundle"].unlink() - assert doctor_report(workspace)["ok"] is True # the pair is not + assert doctor_report(workspace)["ok"] is True # residual: the pair is not + bound_workspace["record"].unlink() + bound_workspace["bundle"].unlink() + assert "missing_bound_evidence" in _codes(doctor_report(bound_workspace["workspace"])) + +def test_rewriting_the_evidence_record_is_caught_only_when_the_chain_binds_it(tmp_path, + bound_workspace): + """Was `test_rewriting_the_evidence_record_to_independent_identities_leaves_doctor_clean_pinned`; + the detection half arrived with chain-bound evidence in the evidence-wiring release. -def test_rewriting_the_evidence_record_to_independent_identities_leaves_doctor_clean_pinned(tmp_path): - """The record is a plain file OUTSIDE the hash chain. A worker with write access - to .loop/ can rewrite it after the fact and doctor will not notice. Binding the - record digest into the chain is Slice 3. + The RESIDUAL is unchanged and kept: a record written by the writer API with no event + store behind it is still a plain file, and a rewrite of it is still invisible. A + record a dispatch bound is not — its bytes are covered by `event_hash`, so the + rewrite surfaces as `evidence_chain_mismatch`. """ workspace = _ws(tmp_path) written = _write(workspace, executor="solo", verifier_identity="solo") @@ -89,12 +139,23 @@ def test_rewriting_the_evidence_record_to_independent_identities_leaves_doctor_c record = json.loads(written["evidence"].read_text(encoding="utf-8")) record["verified_by"]["by"] = "ci" written["evidence"].write_text(json.dumps(record), encoding="utf-8") - assert doctor_report(workspace)["ok"] is True + assert doctor_report(workspace)["ok"] is True # residual + bound = bound_workspace["record"] + data = json.loads(bound.read_text(encoding="utf-8")) + data["verified_by"]["by"] = "ci" + bound.write_text(json.dumps(data), encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(bound_workspace["workspace"])) def test_hand_written_record_with_a_fabricated_code_digest_is_doctor_clean_pinned(tmp_path): """Digests are values the WRITER asserts. Doctor validates their shape, never their truth — a hand-written record is indistinguishable from a runner-written one. + + Narrower since the evidence-wiring release, and deliberately still true: what IS + now checked is the bundle's BYTES (the record's `sha256` must hash the file its + `uri` names) and, for a task still declared in TASKS.json, the recorded + `policy_digest` against the live goalpost. `code_digest` remains unchecked, and + this record names a task the scaffold's TASKS.json does not declare. """ workspace = _ws(tmp_path) written = _write(workspace) @@ -108,29 +169,44 @@ def test_hand_written_record_with_a_fabricated_code_digest_is_doctor_clean_pinne assert doctor_report(workspace)["ok"] is True -def test_no_automated_digest_comparison_exists_pinned(tmp_path): - """Two records for the same task with DIFFERENT policy digests: doctor is silent. - Nothing in this release compares a recorded digest against anything. +def test_policy_digest_comparison_fires_only_against_a_live_task_entry(tmp_path): + """Was `test_no_automated_digest_comparison_exists_pinned`; the comparison arrived in + the evidence-wiring release. + + The RESIDUAL is its two silences, both kept here: a record naming a task TASKS.json + no longer declares is never compared (a rename or removal is a replan, not a + forgery), and neither is any record that is not the latest for its task. Both + records below move the goalpost, and doctor stays quiet until the task is live. """ workspace = _ws(tmp_path) - _write(workspace, iteration_id=1, task=_task()) + _write(workspace, iteration_id=1, task=_task()) # T-1 is not in TASKS.json _write(workspace, iteration_id=2, task=_task(verify="true")) digests = {json.loads((workspace / ".loop" / "evidence" / f"evidence-iter{n}.json") .read_text(encoding="utf-8"))["verified_by"]["policy_digest"] for n in (1, 2)} - assert len(digests) == 2 # the goalpost demonstrably moved - assert doctor_report(workspace)["ok"] is True # and nothing surfaced it + assert len(digests) == 2 # the goalpost demonstrably moved + assert "policy_digest_mismatch" not in _codes(doctor_report(workspace)) # residual + (workspace / "TASKS.json").write_text(json.dumps( + {"schema": "loop-engineer/tasks@1", "project": "p", "tasks": [_task(status="pending")]}), + encoding="utf-8") + assert "policy_digest_mismatch" in _codes(doctor_report(workspace)) + +def test_doctor_hash_verifies_the_referenced_bundle(tmp_path): + """Was `test_doctor_does_not_hash_verify_the_referenced_bundle_pinned`; doctor gained + the composition in the evidence-wiring release. -def test_doctor_does_not_hash_verify_the_referenced_bundle_pinned(tmp_path): - """Slice 2 checks structure, declared independence, and record presence only.""" + Slice 2 checked structure, declared independence and record presence only, and the + hash check existed but nothing called it. Doctor now composes `verify_evidence` over + every discovered record and surfaces its `hash_mismatch` verbatim. + """ workspace = _ws(tmp_path) written = _write(workspace, passed=False) written["bundle"].write_text(json.dumps({"outcome": "PASS", "passed": True}), encoding="utf-8") record = json.loads(written["evidence"].read_text(encoding="utf-8")) - # control: the hash check EXISTS and catches this swap — doctor just never calls it. + # the standalone verb and the doctor sweep now agree, and doctor says it in its issues assert verify_evidence(record, workspace_root=workspace)["ok"] is False - assert doctor_report(workspace)["ok"] is True + assert "hash_mismatch" in _codes(doctor_report(workspace)) def test_code_digest_is_null_for_the_common_python_m_pytest_command_pinned(tmp_path): diff --git a/scripts/test_verifier_identity.py b/scripts/test_verifier_identity.py index 53ccfef..b6e20f5 100644 --- a/scripts/test_verifier_identity.py +++ b/scripts/test_verifier_identity.py @@ -67,6 +67,20 @@ def test_code_digest_says_unresolvable_when_resolution_raises(tmp_path): assert verifier_code_digest("./a.sh", tmp_path) == (None, "unresolvable") +def test_code_digest_says_unresolvable_when_the_WORKSPACE_ROOT_cannot_be_resolved(tmp_path): + """The workspace root is resolved inside the SAME try as argv[0], and must stay there. + + Both interpreter behaviours land on `unresolvable`: on <=3.12 `Path.resolve()` raises + pathlib's RuntimeError for a symlink loop, while 3.13 returns the loop unresolved and + `os.stat` raises ELOOP as an OSError. Narrowing that except boundary to argv[0] alone + is exactly how Slice 2's Task-1 Critical was born, so the root variant is pinned too. + """ + root, other = tmp_path / "root-a", tmp_path / "root-b" + root.symlink_to(other) + other.symlink_to(root) # ELOOP on resolving the workspace root itself + assert verifier_code_digest("./scripts/verify-fast.sh", root) == (None, "unresolvable") + + def test_code_digest_is_null_and_explained_when_the_file_cannot_be_read(tmp_path, monkeypatch): _script(tmp_path, "scripts/verify-fast.sh") monkeypatch.setattr(verifier, "_digest_file", lambda path: None) From 03450906ba5ea95e74e56b80a3a547a1c9cff2e9 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 17:29:14 -0400 Subject: [PATCH 12/22] docs(reference): bind evidence into the chain, move the tiers, name what still is not caught --- README.md | 5 +- loop/evidence.py | 20 +- reference/repo-os-contract.md | 328 +++++++++++++++++++++++++----- reference/safety-and-approvals.md | 2 + schemas/evidence.schema.json | 2 +- 5 files changed, 297 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index f72ecee..51100ca 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,10 @@ ships, on disk and runnable today: named reason otherwise — e.g. `python3 -m pytest`), policy digest — and `loop doctor` fails when a record declares that its own producer verified it; identity is recorded, not proven: a worker can write a - false verifier name, and the records live outside the hash chain. + false verifier name. A verified dispatch **binds its evidence digests into the + hash chain**, and `loop doctor` re-hashes them — binding makes tampering + detectable against an anchor, not impossible: without `--expect-chain-head` a + worker who can rewrite `.loop/` can rewrite the chain too. ![The inspector scores a self-asserted DIY loop 0/weak, then the gate-backed example 90/strong — both runs live](docs/demo.gif) diff --git a/loop/evidence.py b/loop/evidence.py index 17f3957..f66d3df 100644 --- a/loop/evidence.py +++ b/loop/evidence.py @@ -1,11 +1,21 @@ """loop-engineer/evidence@1 — hashed evidence + artifact provenance. ``loop doctor`` discovers and validates records from the declared location -``.loop/evidence/*.json`` (reference/repo-os-contract.md #17) and reports -``self_verified_evidence`` / ``missing_evidence_record``. It does NOT yet -hash-verify the artifacts those records reference, and it never compares a -recorded digest against anything — ``verify_evidence()`` below is the explicit, -caller-invoked check. That wiring is the evidence-wiring slice. +``.loop/evidence/*.json`` (reference/repo-os-contract.md #17), reports +``self_verified_evidence`` / ``missing_evidence_record``, and — since the +evidence-wiring release — composes ``verify_evidence()`` below over every +structurally-valid record, so a referenced artifact that does not hash to the +digest its record declares fails doctor as ``hash_mismatch``. Doctor also +compares the latest record per task against the live TASKS.json goalpost +(``policy_digest_mismatch``), and re-hashes whatever an event bound into the +chain (``evidence_chain_mismatch`` / ``missing_bound_evidence``). + +What is still NOT checked here: ``code_digest`` is never re-hashed against the +verifier file (a verify script legitimately changes between runs, and an +unbaselined comparison would fire on every honest edit), and none of this proves +provenance — a hand-written record whose pointer resolves and whose digests are +self-consistent is indistinguishable from one a dispatch produced. Verification +proves the pointer, never the producer. """ from __future__ import annotations diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index a905d0c..c7ae28d 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -622,9 +622,15 @@ exception. `VerifierExecutionError`, `VerifierNotImplementedError`, and attempted. **Event types:** `contract_opened | iteration_appended | receipt_appended | -terminal_written` — one-to-one with `loop.emit`'s four writer operations +terminal_written | terminal_superseded | approval_requested | +approval_resolved | run_paused | run_resumed` — nine members, matching +`loop.events.EVENT_TYPES` and the `type` enum of `schemas/event.schema.json`. +The **first four** correspond to `loop.emit`'s writer operations (`open_contract`/`append_iteration`/`append_receipt`/`terminate`), so a -future write-through migration targets an already-matching payload shape. +future write-through migration targets an already-matching payload shape. The +remaining five have no `emit` writer by design: `terminal_superseded` is the +administrative correction of §18, and the four run-control types are §19's +projection primitives. **Two-layer enforcement, deliberately split:** the store validates event@1 envelope/payload *shape* only (`loop/events.py::validate_event`, both @@ -751,6 +757,19 @@ appended after a chained prefix is reported as `event_chain_broken` and is unrepairable, because UPDATE is trigger-blocked. Pin your loop-engineer (and action) version per store. +**Bound artifacts (v0.11.0+).** `artifact_hashes` is field eleven of the +twelve-field preimage, so any `{path, sha256}` an appender places there is +already covered by `event_hash` and therefore by `--expect-chain-head`. Binding +evidence needs **no new event type and no second append**: since the +evidence-wiring release a verified dispatch binds three entries on the one +`iteration_appended` event it already writes — the verify bundle, its evidence@1 +record, and the bundle's content-addressed object (§17). The digests are +computed **before** the append and the files are written **after** it. An event +that binds nothing — every event written before that release, and every append +by a foreign writer — is silent by construction, and stays silent: the +append-only triggers forbid the UPDATE a retroactive binding would need, exactly +as `loop migrate` refuses to backfill chain hashes (§22). + ### Integrity boundary The chain is **tamper-evident relative to an anchored head**. That is a @@ -785,7 +804,13 @@ trailing events leaves a shorter but internally valid chain. `deterministic` and `legal_sequence` stay `true`. A future standalone event-store cross-check — a new issue code appended directly to `issues`, as `chain_columns_missing` is — would not move any of those, so it must be added - to this pin's assertions when it is introduced.) + to this pin's assertions when it is introduced. The evidence-wiring release + introduced exactly two such codes, `evidence_chain_mismatch` and + `missing_bound_evidence`; this pin's assertions were **not** widened to name + them. The stronger statement lives in the binding suite instead: + `scripts/test_adversarial_evidence_binding.py::test_a_full_rewrite_of_artifacts_and_store_is_not_caught_without_an_anchor_pinned` + rewrites the artifacts, the object, the bound digests and the chain, and + asserts the whole report is `ok` with **zero** issues.) - **A chain-column downgrade.** Dropping `event_hash`/`prev_event_hash`, or rebuilding the store without them, silently downgrades a chained history to an unchained one. An unchained or legacy doctor report is *not* proof of @@ -835,10 +860,15 @@ workspace, rejects traversal and symlink escapes, and verifies the file hash. `.loop/artifacts/objects/` without writing it. **Scope boundary:** `loop doctor` reads evidence@1 records from the declared -location `.loop/evidence/*.json` and validates them; it does **not** yet -hash-verify the artifacts they reference, does **not** compare any recorded -digest against anything, and `Succeeded` still requires non-empty evidence -*paths*, not verified hashes. Those tightenings are the next slice. +location `.loop/evidence/*.json`, validates them, **hash-verifies the artifact +each one references**, and compares the `policy_digest` of the **latest record +per task** against the live `TASKS.json` entry. Under the default +`all_required` completion policy `Succeeded` still requires non-empty evidence +*paths*; under the opt-in `all_required_verified_evidence` policy it requires +every entry to be a hash-verified evidence@1 record that — wherever an event +store exists to bind against — some event bound into the hash chain, and whose +recorded goalpost is the live one. `code_digest` is still never re-hashed +against the verifier file, and no check here proves who produced a record. **Artifact provenance:** `kind` remains an open vocabulary (for example, `verify-bundle`, `log`, `diff`, `screenshot`, or `report`), while `produced_by` @@ -889,8 +919,10 @@ make the digest noise; `id` binds *which* goalpost the digest names, and `depends_on` binds its declared ordering, so both are included. The digest binds the criterion **reference**, not the criterion **text** — -editing `SPEC.md`'s acceptance wording leaves it unchanged. Binding criterion -text is the evidence-wiring slice. +editing `SPEC.md`'s acceptance wording leaves it unchanged. The evidence-wiring +release did **not** change that: criterion text remains bound by nothing, pinned +by `test_criterion_text_is_still_unbound_pinned`. Binding evidence bytes does +not bind intent. **Conformance vector.** Over the task entry @@ -923,14 +955,25 @@ to the bundle bytes via `sha256`). An evidence record MUST NOT be named bundle with no green marker, i.e. a phantom failing gate. `verifier.source` is `declared_command` only when the task's declared `verify` command was executed. A bundle whose source is `injected_callable` carries a -caller-supplied verdict and is not gate evidence. `scripts/metrics.py` does not -read `verifier.source` today, so an injected-callable bundle counts toward FCR -exactly like a declared-command bundle — enforcing that distinction is Slice 3 -scope. Before this slice the runner wrote no bundles at all, so a -runner-driven contract's FCR input set changes with this release. `loop -simulate` predicts decisions, not writes: it reports `legacy_sync_would_write` -because that write is conditional, but it does not enumerate the bundle and -record a dispatch always writes. +caller-supplied verdict and is not gate evidence. `scripts/metrics.py` enforces +that distinction: a bundle whose `verifier.source` is **explicitly** +`injected_callable` is excluded from the FCR input set before any metric is +derived, and the excluded filenames are listed under +`provenance.injected_verifier_bundles` so the exclusion is visible rather than a +silent FCR shift. A bundle carrying no `verifier` block at all has an *unknown* +source and still counts — grandfathering by absence, not by guess. `source` is a +string the writer declares, not a fact metrics can check, so hand-writing +`declared_command` restores gate-evidence status; that limitation is pinned by +`test_metrics_still_counts_a_hand_written_declared_command_bundle_pinned`. +Two input-set changes land together in this release and are stated rather than +absorbed: before it the runner wrote no bundles at all, and injected-callable +bundles counted — so a runner-driven contract's FCR input set moves on upgrade. +`loop simulate` predicts decisions, not writes: it reports +`legacy_sync_would_write` because that write is conditional, but it does not +enumerate the bundle, record and content-addressed object a dispatch always +writes, and it does not enumerate the object store. A boolean that is `True` on +every dispatch carries no information, so `_empty_prediction` gains no field +here (§20). **The partition.** `visible` defaults to the task's `criterion_ref`; `holdout` is empty unless the task declares `holdout_criteria`; both fields @@ -948,40 +991,195 @@ fails. On the `loop run` path both identities are operator-supplied (`--executor`, `--verifier-identity`); their defaults (`unattributed`, `loop.run`) never collide, so a default run cannot manufacture the finding. -**The integrity boundary, in four honest tiers** — not a single "surfaces / -does not surface" pair: +### Bound evidence (v0.11.0+) + +**The bound set, in the order the writer emits it.** A verified dispatch binds +exactly three `{path, sha256}` entries into the `artifact_hashes` of the one +`iteration_appended` event it already writes (§16), in this order: + +1. `.loop/artifacts/verify-iter.json` — the verify bundle, at the bundle's + own sha256; +2. `.loop/evidence/evidence-iter.json` — the evidence@1 record, at the + record file's sha256; +3. `.loop/artifacts/objects//` — the content-addressed **object**, a + byte-identical copy of the bundle, at the bundle's sha256 again. + +**The object's location is derived, never declared.** It is +`artifact_object_path(workspace, record["sha256"])` — +`.loop/artifacts/objects//` — a pure function of a +digest evidence@1 already carries, so **evidence@1 gains no field** and a +third-party reader locates the object from `record["sha256"]` alone. The object +is the recovery source when the friendly-named bundle is swapped: the swap fails +doctor as `evidence_chain_mismatch` *and* the original bytes are still on disk. +Deleting the object first does not clear the path — the object is itself a bound +artifact, so its removal is reported as `missing_bound_evidence`. + +**Objects are created once and never overwritten.** The write uses the same +hard-link create-once primitive as the immutable terminal record. An existing +object with identical bytes is idempotent success (a re-run of the same +dispatch); an existing object with **different** bytes for the same digest is a +typed `EmitError` naming the digest, because a corrupted or colliding object +store must never be silently accepted. + +**Ordering: build (no I/O) → append (binds) → write (object, staged bundle, +record, replace).** The builder is pure — it renders the exact bytes and their +digests and touches nothing — so the event commits to digests that describe what +is about to land. Two consequences, both deliberate: a **build failure commits +nothing** (a non-canonicalizable task or an invalid record is refused before any +file exists), and a **crash after the durable append** leaves an event naming +artifacts that are absent — reported as `missing_bound_evidence` rather than +passing as a silent gap. The crash window still exists; what changed is that it +is no longer invisible. + +### The verified-evidence completion mode (v0.11.0+) + +`completion_policy.mode` accepts a second value, +`all_required_verified_evidence`, alongside the default `all_required`. The +criteria half is unchanged — every declared criterion must still be `True`; the +mode raises the **evidence** bar only. It is **opt-in and never retroactive**: +`normalize_completion_policy(None)` still returns `{"mode": "all_required"}`, so +every record written before this release — including both shipped examples, +which declare no policy at all — keeps the old bar forever +(`test_the_verified_evidence_mode_is_opt_in_and_not_retroactive_pinned`). + +**Four enforcement layers, each as strong as it can honestly be.** + +| Layer | Reaches | Enforces | +|---|---|---| +| `emit.terminate` (write time) | workspace + event store | the full bar; failure is a typed `EmitError` | +| `loop doctor` / `loop.contract` (read time) | workspace + event store | the same bar, reported as `unverified_evidence_terminal` | +| `loop.reducer` (pure fold) | nothing — no I/O | shape only: every entry is a workspace-relative `.loop/evidence/*.json` path | +| `loop.integrations.to_terminal_state` (pure projection) | nothing — no workspace | the same shape half, returning `FailedUnverifiable` rather than raising | + +The two workspace-bearing layers share **one predicate** +(`loop.contract._strict_evidence_failure`); `emit.terminate` imports it rather +than restating it, because two hand-written copies of a three-part check drift +and a drift here is a silent false completion. That predicate is a strict +**superset** of the pure half: it applies the reducer's own +`evidence_entry_is_record_shaped` to the cited entry first, so the writer can +never accept a terminal that its own replay would refuse. The two pure layers +check shape and nothing else because they hold no filesystem — making them +*appear* to enforce binding would be the pretending this contract refuses. + +**What the mode proves, exactly.** For every cited entry: (1) the entry resolves +to a readable evidence@1 record that validates, and the artifact its `uri` names +resolves inside the workspace and hashes to the digest the record declares; +(2) **when an event store exists**, those record bytes are the bytes some event +bound into the chain, at that digest; and (3) when the record names a task still +in `TASKS.json`, its recorded `policy_digest` equals the live goalpost. A goalpost +that cannot even be **computed** is a failure, not a skip — unestablished is not +agreement. An event store that cannot be **read** is likewise a refusal, never a +pass. The message names which of the three sub-checks failed. + +**What it does not prove, stated without hedging.** It does not prove that a +human, a runner, or anything at all *produced* the record: a hand-written record +whose pointer resolves and whose digests are self-consistent satisfies the mode +in a **store-less** contract, because there is nothing there to bind against +(`test_the_strict_mode_accepts_a_hand_written_record_in_a_store_less_contract_pinned`). +It does not prove the chain itself was not rewritten: a worker who can rewrite +`.loop/` can rewrite the store, so **without `--expect-chain-head` a full rewrite +of artifacts *and* chain satisfies the strict mode** +(`test_a_full_rewrite_satisfies_the_strict_mode_without_an_anchor_pinned`). The +mode's strength is therefore **store-dependent and anchor-dependent**. + +**Deleting the store is not louder — it is quieter.** Measured, not assumed: a +sidecar-free read (v0.10.0) leaves no `-wal`/`-shm` behind, and doctor's +`missing_event_store` tripwire is gated on exactly that residue, so removing +`.loop/events.db` after an ordinary run leaves plain `loop doctor` at `ok: true` +with **zero issues**, and `emit.terminate` then accepts a hand-written record +under the strict mode. `missing_event_store` catches only the sloppier deletion +that leaves a sidecar. The control that does hold is the external anchor: +`loop doctor --expect-chain-head ` reports `chain_anchor_mismatch` against +the deleted store and stays quiet against an intact one. Without an external +anchor, deleting the store silently disables this mode's chain-boundness check. + +**A deliberate asymmetry with doctor's general goalpost check.** Doctor compares +the **latest record per task** (see the tier list below); the terminal check +compares **every cited record**. Citing a record in a `Succeeded` terminal is a +present-tense claim that *this* record backs completion now, so a stale goalpost +in a cited record is a false completion, not an artifact of history. + +**`loop run`'s auto-terminal adopts the mode only when it can satisfy it**, and +never on file existence: it evaluates the same shared predicate against the very +records it would cite, **before** appending the terminal event. When it cannot, +it writes `{"mode": "all_required"}` with `evidence: ["RUNLOG.md"]` and says why +in `reason` — either `tasks with no evidence record: …` or `evidence records do +not meet the verified-evidence bar: `. Downgrading silently +would be a self-serving choice; both branches are pinned. + +### The integrity boundary, in four honest tiers + +Not a single "surfaces / does not surface" pair: - **Fails `loop doctor`:** a record declaring self-verification (`self_verified_evidence`); a runner-written bundle whose record is absent (`missing_evidence_record`); a malformed or unparseable record - (`invalid_evidence` — an errored check fails, it never skips). + (`invalid_evidence` — an errored check fails, it never skips). Added by the + evidence-wiring release: a discovered record whose referenced artifact does + not match its `sha256` (`hash_mismatch`) or does not resolve at all + (`missing_evidence_path`, surfaced verbatim out of `verify_evidence`); the + **latest** record for a task whose recorded goalpost disagrees with — or + cannot be compared against — the live `TASKS.json` entry + (`policy_digest_mismatch`); a chain-bound artifact whose bytes changed + (`evidence_chain_mismatch`) or vanished (`missing_bound_evidence`); and a + `Succeeded` terminal under `all_required_verified_evidence` whose cited + evidence fails the bar (`unverified_evidence_terminal`). - **Recorded for later comparison, not checked by any shipped surface:** - `policy_digest` and `code_digest`. Nothing in v0.11.0 compares either - against the live TASKS.json entry or re-hashes the verifier file, so a moved - goalpost or a swapped verify script is *legible in a diff of two records* - and nothing more. Automated comparison is the evidence-wiring slice. -- **Detectable only by explicitly calling `verify_evidence()`:** a swapped - bundle. `loop doctor` does not hash-verify referenced artifacts in this - release. + `code_digest`, and now only `code_digest`. Nothing re-hashes the verifier + file, and nothing will without a declared baseline: a verify script + legitimately changes between runs, so an unbaselined comparison would fire on + every honest edit. A swapped verify script therefore remains *legible in a + diff of two records* and nothing more + (`test_doctor_does_not_re_hash_the_verifier_file_pinned`). **`policy_digest` + has left this tier** — the evidence-wiring release compares it against the + live goalpost. +- **Detectable only by explicitly calling `verify_evidence()`:** **this tier is + now empty.** The evidence-wiring release composes that call inside + `loop doctor` for every discovered record, so a swapped bundle — the one + member this tier ever had — now fails doctor as `hash_mismatch` in the first + tier above. - **Not surfaced at all:** a worker that writes a false `verified_by.by`; a - worker that omits its executor identity; a hand-written record whose - digests are invented (doctor validates their *shape*, never their truth); a - rewrite or deletion of the record **and** its bundle together. **Both - artifacts are plain files outside the hash chain: a worker with write - access to `.loop/` can rewrite or remove them and `loop doctor` will not - notice.** Binding the record digest into the chain requires a new event - type and is the evidence-wiring slice. A record whose referenced bundle is - absent also passes `loop doctor` clean: `missing_evidence_record` is - one-directional (it walks bundles looking for their record, never the - reverse), so doctor never checks that a record's `uri` resolves. And an - iteration committed to the event log with no evidence written at all — the - sanctioned crash window between the durable append and the evidence write — - is undetectable, because the tripwire needs a bundle to fire before it can - notice anything missing. Closing both gaps is Slice 3 scope. + worker that omits its executor identity; a hand-written record whose pointer + resolves and whose digests are self-consistent (verification proves the + *pointer*, never the *provenance* — + `test_a_hand_written_record_pointing_at_a_real_file_still_passes_pinned`); the + *text* of a criterion, which `policy_digest` never covered and still does not. + Plus the residuals this release's own binding leaves, each pinned by name: + - a record written through the **writer API** (`emit.write_verify_evidence`) + appends no event, so nothing binds it and a later rewrite is invisible + (`test_a_record_written_outside_a_dispatch_is_never_chain_bound_pinned`); + - a **legacy** event carries `artifact_hashes: []` and can never be bound + retroactively — the append-only triggers forbid the UPDATE a backfill would + need — so a pre-release iteration whose evidence was deleted stays + undetectable at this layer forever + (`test_a_legacy_iteration_event_can_never_be_bound_retroactively_pinned`); + - an **older** record for a task whose goalpost has since moved is not + compared, because comparing every record would make each honest + re-verification a permanent failure + (`test_an_older_record_for_a_moved_goalpost_is_not_compared_pinned`); + - a hand-written record **satisfies `all_required_verified_evidence` in a + contract that has no event store**, because there is nothing to bind + against — and deleting a store from a run that had one is *quieter*, not + louder: sidecar-free reads leave no residue for the `missing_event_store` + tripwire, so plain doctor reports `ok` with zero issues. The external + anchor is the compensating control + (`test_the_strict_mode_accepts_a_hand_written_record_in_a_store_less_contract_pinned`); + - deleting the artifacts, the records, the store **and** its sidecars leaves a + contract that never ran, and doctor is clean and correct to be — detection + needs something outside the tree + (`test_deleting_every_artifact_and_the_store_leaves_a_clean_doctor_pinned`, + whose own positive control is the anchor); + - and — the headline — **a worker who can rewrite `.loop/` can rewrite the + store, so without `--expect-chain-head` a full rewrite of artifacts *and* + chain verifies clean, including under the strict completion mode** + (`test_a_full_rewrite_of_artifacts_and_store_is_not_caught_without_an_anchor_pinned`, + `test_a_full_rewrite_satisfies_the_strict_mode_without_an_anchor_pinned`). This does not prove independence. It surfaces **declared** self-verification, -and it records — honestly, with nulls where the process could not know — -what verified the work. +and it records — honestly, with nulls where the process could not know — what +verified the work. This does not make evidence tamper-proof. It binds evidence +bytes into a chain someone outside the worker's trust domain can anchor, and it +fails closed when what is on disk is not what was verified. --- @@ -995,12 +1193,11 @@ non-empty `justification`, and `{by, at}` `authority`, and its `causation_id` must identify the terminal event it corrects; chained corrections therefore remain auditable without replacing any record. -**Scope boundary:** this is a fifth event type with no corresponding -`loop.emit` writer operation — deliberately: unlike the other four, -`terminal_superseded` is administrative and event-log-only; §16's -“one-to-one with `loop.emit`'s four writer operations” describes the other four -types and predates this addition. It is not file-based `terminal@1` replacement -or an `emit`/`doctor` workflow. +**Scope boundary:** this event type has no corresponding `loop.emit` writer +operation — deliberately: unlike the first four types, `terminal_superseded` is +administrative and event-log-only (§16's type list records which four types map +to writer operations and which five do not). It is not file-based `terminal@1` +replacement or an `emit`/`doctor` workflow. **Domain enforcement:** the EventStore validates envelope and payload shape only; the reducer alone admits this type after a terminal, verifies its @@ -1105,9 +1302,12 @@ only finding is the anchor. A present, readable store adds `"event_store": {"present": true, "readable": true, "run_id", "event_count", "state_json_agrees", "deterministic", "legal_sequence", "chain"}`; any of `state_field_mismatch`, `desynced_terminal_window`, `terminal_state_mismatch`, -`illegal_event_sequence`, `event_chain_broken`, `chain_columns_missing`, or -`chain_anchor_mismatch` fails doctor (`ok: false`) with the identical issue -code the `status`/`replay` verbs already use. A store that cannot be read at +`illegal_event_sequence`, `event_chain_broken`, `chain_columns_missing`, +`evidence_chain_mismatch`, `missing_bound_evidence`, or +`chain_anchor_mismatch` fails doctor (`ok: false`). The findings that come from +the composed verbs keep the identical issue code those verbs already use; +`chain_columns_missing`, `evidence_chain_mismatch` and `missing_bound_evidence` +are doctor's own store-gated checks, appended to the same list. A store that cannot be read at all — `corrupt_store`, `empty_store`, `invalid_event`, or `ambiguous_run_id` — also fails doctor rather than being silently skipped; `"event_store"` reports `{"present": true, "readable": false, "error_code": }` in that case. @@ -1139,13 +1339,35 @@ it is the honest statement of how much of the log the chain does not cover. | `missing_event_store` | `events.db` is absent but `-wal`/`-shm` sidecars remain — the store was deleted. Distinct from the pre-existing `missing_store`, which `status`/`replay`/`run`/`migrate` raise when a verb that *requires* a store is pointed at a workspace that has none; `missing_event_store` is a doctor finding about a store that evidently once existed. | | `self_verified_evidence` | A discovered evidence@1 record declares `produced_by.executor == verified_by.by` (strip+casefold) — the producer verified its own work. Enforces the independence rule of `reference/safety-and-approvals.md` §5, which was prose-only before v0.11.0. | | `missing_evidence_record` | A runner-written verify bundle `.loop/artifacts/verify-iter.json` exists with no matching `.loop/evidence/evidence-iter.json`. Residue of a removed provenance record, in the same family as `missing_event_store`. Fires only when a bundle is present, so an absent-everything contract stays byte-identical. | +| `evidence_chain_mismatch` | An artifact whose digest an event bound into the hash chain no longer matches its bytes on disk. The message names the digest the chain committed and the digest found; the original bytes may still remain in the content-addressed object store at `.loop/artifacts/objects//`. | +| `missing_bound_evidence` | An event bound an artifact path into the chain and that path is now absent or unreadable — a deleted bundle/record pair, a deleted object, or the sanctioned crash window between the durable append and the evidence write. Fires only for events that bound something: a legacy event binds nothing and is silent by construction. | +| `policy_digest_mismatch` | The **latest** evidence record for a task records a `policy_digest` that is not the live `TASKS.json` goalpost — the declared `verify`/`criterion_ref`/`depends_on`/`id` changed after the verification. Also fires when the live entry cannot be canonicalized at all, because a comparison that cannot run has not passed. Re-verify to record the current goalpost. | +| `unverified_evidence_terminal` | A `Succeeded` terminal declaring `completion_policy.mode: all_required_verified_evidence` has an evidence entry that fails one of the mode's three checks: it is not a hash-verified evidence@1 record; or (when an event store exists) no event bound those bytes into the chain; or the record's `policy_digest` is not the live `TASKS.json` goalpost for the task it names. The message names which. An unreadable store is also a failure. In a contract with **no** event store the middle check is skipped — the mode's strength is store-dependent by construction (§17). | + +**Do not confuse `missing_evidence_record` with `missing_evidence_path`.** They +are opposite directions of the same pair. `missing_evidence_record` walks +*bundles* looking for the record that should describe them. +`missing_evidence_path` comes straight out of `verify_evidence()` and means a +*record's* `uri` does not resolve. Both can be true at once and neither implies +the other. **Evidence discovery (v0.11.0+).** `loop doctor` scans `.loop/evidence/*.json` when the directory exists; an absent directory with no runner bundle is a no-op that leaves every doctor key byte-identical (no new top-level key was added). A malformed or unparseable record **fails** doctor rather than being skipped, and `loop-engineer/evidence@1` joins `schemas_checked` when at least -one record was read. +one record was read. Since the evidence-wiring release each structurally-valid +record is additionally passed to `verify_evidence()` — surfacing +`hash_mismatch`, `missing_evidence_path`, `workspace_escape`, `not_a_file` and +`invalid_uri` **verbatim**, per the rule that doctor composes a verb and reuses +its issue codes rather than inventing parallel ones — and the latest record per +task is compared against the live goalpost (`policy_digest_mismatch`). The +`verify_evidence()` call is guarded on the record having no structural issues, +so a malformed record is reported once, not twice. The chain-binding walk is a +separate, store-gated check that lives beside `chain_columns_missing` in the +event-store block above; it costs one extra read-only fold of the store, because +it needs envelope-level `artifact_hashes` that no report returns and widening a +report's dict would leak into the `loop status` / `loop replay` CLI JSON. **`loop migrate`.** `loop migrate ` is the only store-upgrade path: explicit, idempotent, and non-rewriting. It widens `events` with the two diff --git a/reference/safety-and-approvals.md b/reference/safety-and-approvals.md index ad215f5..0cbdd04 100644 --- a/reference/safety-and-approvals.md +++ b/reference/safety-and-approvals.md @@ -98,6 +98,8 @@ This is why two structural rules from [[loop-repair]] are load-bearing safety ru Since v0.11.0 this invariant has a machine check: `loop doctor` reports `self_verified_evidence` when an evidence@1 record declares that its producer also verified it (`reference/repo-os-contract.md` §17). The check surfaces *declared* self-verification — a worker that writes a false verifier name, or that rewrites the record afterwards, is not caught, which is why the protected-file and canary rules above remain load-bearing. +The goalpost half of the invariant now has a machine check too: `loop doctor` reports `policy_digest_mismatch` when the latest evidence record for a task records a goalpost — `id` / `verify` / `criterion_ref` / `depends_on` — that is not the live `TASKS.json` entry, so moving the goalpost after a verification is a finding rather than a diff nobody reads. It binds the criterion *reference*, never the criterion *text*, and a rename still evades it (`_self_verified` compares declared identities, and a record naming a task no longer in `TASKS.json` is not compared at all) — which is exactly why the protected-file rule and the canaries of §6 stay load-bearing rather than being replaced by it. + --- ## 6. Anti-cheat canaries diff --git a/schemas/evidence.schema.json b/schemas/evidence.schema.json index 18451ff..b861f92 100644 --- a/schemas/evidence.schema.json +++ b/schemas/evidence.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "loop-engineer/evidence@1", "title": "Loop Engineer Evidence @1", - "description": "A hashed evidence record with artifact provenance. loop doctor discovers and validates records from the declared location .loop/evidence/*.json; it does not yet hash-verify the artifacts they reference. See reference/repo-os-contract.md #17.", + "description": "A hashed evidence record with artifact provenance. loop doctor discovers records from the declared location .loop/evidence/*.json, validates them, hash-verifies the artifact each one references, and compares the latest record per task against the live TASKS.json goalpost; a verified dispatch also binds the record's bytes into the event hash chain. Hash verification proves the pointer, never the provenance, and code_digest is never re-hashed against the verifier file. See reference/repo-os-contract.md #17.", "type": "object", "required": ["schema", "id", "kind", "uri", "sha256", "media_type", "produced_by", "created_at"], "properties": { From 535a2c2c5bb66cd6327b3c6f7e2a6851b8a59a7a Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 17:58:05 -0400 Subject: [PATCH 13/22] test(conformance): pin the documented binding vector and the retired claims --- scripts/test_conformance.py | 103 ++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/scripts/test_conformance.py b/scripts/test_conformance.py index d26f389..33c8826 100644 --- a/scripts/test_conformance.py +++ b/scripts/test_conformance.py @@ -436,3 +436,106 @@ def test_documented_policy_digest_vector_matches_the_implementation(): assert verification_policy_digest(_POLICY_VECTOR_TASK) == _POLICY_VECTOR_DIGEST doc = (Path(__file__).resolve().parent.parent / "reference" / "repo-os-contract.md").read_text(encoding="utf-8") assert _POLICY_VECTOR_DIGEST in doc and _POLICY_VECTOR_CANONICAL in doc + + +# --------------------------------------------------------------------------- # +# Evidence-wiring doc pins — the shipped surfaces and the normative doc cannot +# drift apart. Every assertion below was proven to FAIL against the tree as it +# stood before the documentation commit; a pin that passes both before and +# after documents nothing. +# --------------------------------------------------------------------------- # + +from loop import emit # noqa: E402 +from loop.verifier import injected_verifier_identity # noqa: E402 + +_CONTRACT_DOC = ROOT / "reference" / "repo-os-contract.md" + +# The retired sentence was written three different ways on the three surfaces +# ("does NOT yet hash-verify", "does not yet hash-verify", "does not +# hash-verify"), so the ban is case-insensitive and the "yet" is optional. A +# case-sensitive substring ban passes against the uppercase one and proves +# nothing about it. +_RETIRED_HASH_VERIFY_CLAIM = re.compile(r"does\s+not\s+(?:yet\s+)?hash-verify", re.IGNORECASE) + + +def _binding_fixture(tmp_path: Path) -> Path: + """A minimal in-flight contract that ``build_verify_evidence`` can render + against — it needs only a readable ``.loop/state.json``.""" + return _scaffold_inflight(tmp_path / "binding") + + +def _bound_evidence_section() -> str: + """§17's ``### Bound evidence`` subsection, heading to next heading.""" + text = _CONTRACT_DOC.read_text(encoding="utf-8") + assert "### Bound evidence" in text, "§17 does not document the bound evidence set" + start = text.index("### Bound evidence") + end = text.find("\n### ", start + 1) + return text[start:] if end == -1 else text[start:end] + + +def test_documented_artifact_binding_vector_matches_the_writer(tmp_path): + """The three bound paths and their order are normative, not incidental.""" + workspace = _binding_fixture(tmp_path) + built = emit.build_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_POLICY_VECTOR_TASK, passed=True, + code_identity=injected_verifier_identity()) + # Order is the WRITER'S CONVENTION, not a correctness property — the doctor + # walk builds a path->digest map and `event_hash` recomputes over the stored + # list exactly as written, so any order verifies. It is pinned to the SHIPPED + # order (bundle -> record -> object) purely so it cannot drift silently away + # from the ordered list §17 publishes. + assert [entry["path"] for entry in built.artifact_hashes] == [ + ".loop/artifacts/verify-iter1.json", + ".loop/evidence/evidence-iter1.json", + f".loop/artifacts/objects/{built.sha256[:2]}/{built.sha256}", + ] + # And the doc's numbered list must enumerate the same three in the same order. + section = _bound_evidence_section() + markers = (".loop/artifacts/verify-iter", ".loop/evidence/evidence-iter", + ".loop/artifacts/objects/") + for marker in markers: + assert marker in section, f"§17 bound-evidence set omits {marker}" + positions = [section.index(marker) for marker in markers] + assert positions == sorted(positions), ( + f"§17 lists the bound set out of writer order: {positions}") + + +def test_every_new_doctor_code_is_documented_in_section_22(): + doc = _CONTRACT_DOC.read_text(encoding="utf-8") + for code in ("evidence_chain_mismatch", "missing_bound_evidence", + "policy_digest_mismatch", "unverified_evidence_terminal"): + assert f"`{code}`" in doc, f"§22 does not document {code}" + + +def test_no_shipped_surface_still_claims_evidence_is_unverified(): + for path in (ROOT / "loop" / "evidence.py", ROOT / "schemas" / "evidence.schema.json", + _CONTRACT_DOC): + text = path.read_text(encoding="utf-8") + assert not _RETIRED_HASH_VERIFY_CLAIM.search(text), ( + f"{path.name} still disclaims hash verification") + # Whitespace-normalised: the retired tier bullet wrapped between the "**" + # and `policy_digest`, so a raw substring ban never matched it either. + normalised = " ".join(text.split()) + assert "not checked by any shipped surface:** `policy_digest`" not in normalised, ( + f"{path.name} still files policy_digest under the unchecked tier") + # decision 14: the strict mode's two residuals must be named where the mode is + # documented, not only in this plan. A capability paragraph without its limits is + # the overclaim this slice exists to prevent. + contract = " ".join(_CONTRACT_DOC.read_text(encoding="utf-8").split()) + assert "no event store" in contract and "--expect-chain-head" in contract + # §17 closes with the trust-domain sentence rather than the plan's + # "detectable against an anchor" wording (that phrasing ships in the README). + # Pinned against the sentence the contract actually carries — absent before + # this release, so the assertion discriminates. + assert "trust domain can anchor" in contract + # Deliberately NOT asserting the absence of "tamper-proof": §17 names it as the + # forbidden claim, so a substring ban would fire on the honesty sentence itself. + + +def test_section_16_event_type_list_matches_the_code(): + from loop.events import EVENT_TYPES + doc = _CONTRACT_DOC.read_text(encoding="utf-8") + # §16 renders the nine members inside one wrapped backtick span, so a bare + # name is as good as a backticked one here. + assert all(f"`{name}`" in doc or name in doc for name in EVENT_TYPES) + assert "one-to-one with `loop.emit`'s four writer operations" not in doc From 6b76a62693f29e1d35ce290f60525b0bf7cdaa77 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 18:15:20 -0400 Subject: [PATCH 14/22] docs(safety): attribute the goalpost rename evasion to the check that has it The parenthetical credited _self_verified, which is the independence check behind self_verified_evidence and plays no part in the goalpost comparison. The rename escape is _policy_digest_issues skipping a task_id that TASKS.json no longer declares. --- reference/safety-and-approvals.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference/safety-and-approvals.md b/reference/safety-and-approvals.md index 0cbdd04..67d8640 100644 --- a/reference/safety-and-approvals.md +++ b/reference/safety-and-approvals.md @@ -98,7 +98,7 @@ This is why two structural rules from [[loop-repair]] are load-bearing safety ru Since v0.11.0 this invariant has a machine check: `loop doctor` reports `self_verified_evidence` when an evidence@1 record declares that its producer also verified it (`reference/repo-os-contract.md` §17). The check surfaces *declared* self-verification — a worker that writes a false verifier name, or that rewrites the record afterwards, is not caught, which is why the protected-file and canary rules above remain load-bearing. -The goalpost half of the invariant now has a machine check too: `loop doctor` reports `policy_digest_mismatch` when the latest evidence record for a task records a goalpost — `id` / `verify` / `criterion_ref` / `depends_on` — that is not the live `TASKS.json` entry, so moving the goalpost after a verification is a finding rather than a diff nobody reads. It binds the criterion *reference*, never the criterion *text*, and a rename still evades it (`_self_verified` compares declared identities, and a record naming a task no longer in `TASKS.json` is not compared at all) — which is exactly why the protected-file rule and the canaries of §6 stay load-bearing rather than being replaced by it. +The goalpost half of the invariant now has a machine check too: `loop doctor` reports `policy_digest_mismatch` when the latest evidence record for a task records a goalpost — `id` / `verify` / `criterion_ref` / `depends_on` — that is not the live `TASKS.json` entry, so moving the goalpost after a verification is a finding rather than a diff nobody reads. It binds the criterion *reference*, never the criterion *text*, and a rename still evades it (a record naming a task no longer in `TASKS.json` is not compared at all) — which is exactly why the protected-file rule and the canaries of §6 stay load-bearing rather than being replaced by it. --- From bfad38c96b99e361362f42fbe13d78ced816f631 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 19:26:04 -0400 Subject: [PATCH 15/22] fix(completion): a cited record must attest a PASS, not merely be authentic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The strict mode verified everything about a cited evidence record except what it said. A dispatch whose verifier FAILED wrote a genuine, hash-verified, chain-bound record of that failure, and emit.terminate(state="Succeeded", ..., completion_policy="all_required_verified_evidence") accepted it: doctor then reported ok with zero issues and the reducer replayed Succeeded. Every layer proved the evidence was real; none asked whether it said the work passed. _strict_evidence_failure gains a verdict sub-check, judged by the repo's one green-marker rule. That rule lived only in scripts/metrics.py (a bundle is green when outcome == "PASS" or passed is True; a score-only bundle reads RED), so it is hoisted to loop.evidence.verify_bundle_is_green and metrics now imports it — a bundle that reads RED to the FCR gate can no longer read GREEN to the completion gate. Decided explicitly: a record whose kind is not verify-bundle carries no verdict this layer can read, and is REFUSED rather than waved through. Unreadable, digest-drifted or unparseable verdicts are refusals, never skips (R007). The pure layers are untouched by design (decision 14): the reducer and the integrations projection hold no filesystem, so they still check shape only. --- loop/contract.py | 57 ++++++- loop/evidence.py | 15 ++ scripts/metrics.py | 5 +- scripts/test_completion_verdict.py | 237 +++++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 scripts/test_completion_verdict.py diff --git a/loop/contract.py b/loop/contract.py index a36e496..22adace 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -791,19 +791,66 @@ def _validate_evidence_records(paths: LoopPaths, mode: str, issues: list[dict]) return checked +def _verdict_failure(record: Mapping[str, Any], paths: LoopPaths) -> str | None: + """Why the record's artifact does not attest a PASS, or ``None`` when it does. + + Hash-verification proves the pointer resolves to the bytes the record committed + to; it says nothing about what those bytes SAY. A record backing ``Succeeded`` + must point at a green verdict, judged by the repo's one green-marker rule + (``loop.evidence.verify_bundle_is_green``) — the same rule ``scripts/metrics.py`` + scores FCR with, so a bundle that reads RED to metrics can never read GREEN here. + + A record whose ``kind`` is anything other than ``verify-bundle`` carries no + verdict this layer can read (evidence@1's ``kind`` is an open vocabulary — a log, + a diff, a screenshot). Such a record is REFUSED rather than waved through: the + strict mode's claim is that completion is backed by a verification that passed, + and an artifact with no verdict cannot make that claim. Unreadable or + unparseable is likewise a refusal, never a skip (R007). + """ + from .evidence import VERIFY_BUNDLE_KIND, verify_bundle_is_green + + kind = record.get("kind") + if kind != VERIFY_BUNDLE_KIND: + return (f"is a {kind!r} record, not a {VERIFY_BUNDLE_KIND}: it carries no verdict, " + f"so it cannot show that anything passed") + uri = record["uri"] + try: + blob = (paths.workspace / uri).read_bytes() + except (OSError, ValueError) as exc: + return f"cites a verify bundle that cannot be read ({exc})" + # Re-anchored to the digest the record declares, so the bytes judged here are the + # bytes hash-verification just accepted rather than whatever landed since. + if hashlib.sha256(blob).hexdigest() != record["sha256"]: + return (f"cites a verify bundle whose bytes changed between hash verification " + f"and the verdict read: {uri}") + try: + bundle = json.loads(blob.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return f"cites a verify bundle that is not UTF-8 JSON ({exc})" + if not isinstance(bundle, dict): + return "cites a verify bundle that is not a JSON object" + if not verify_bundle_is_green(bundle): + return (f"cites a verify bundle whose verdict is not a pass " + f"(outcome={bundle.get('outcome')!r}, passed={bundle.get('passed')!r}) — " + f"evidence of failure cannot back Succeeded") + return None + + def _strict_evidence_failure(entry: object, paths: LoopPaths, bound: Mapping[str, str] | None) -> str | None: """The ONE definition of the verified-evidence bar (plan decision 14). Returns ``None`` when the cited entry can back ``Succeeded``, otherwise a detail - string naming which of the three sub-checks failed: + string naming which of the four sub-checks failed: 1. self-consistency — the entry is a readable evidence@1 record whose ``uri`` resolves inside the workspace and hashes to the digest it declares; - 2. chain-boundness — some event bound this record's path AT its current bytes. + 2. verdict — the artifact it points at is a verify bundle that says PASS. An + authentic record of a FAILING verification is still not proof of success; + 3. chain-boundness — some event bound this record's path AT its current bytes. ``bound is None`` means there is no event store at all, and the sub-check is skipped: the store-less writer-API path is a DOCUMENTED degradation, not a pass; - 3. goalpost agreement — when the record names a task still in TASKS.json, its + 4. goalpost agreement — when the record names a task still in TASKS.json, its recorded ``policy_digest`` equals the live one. A goalpost that cannot even be computed is a FAILURE, not a skip: unestablished is not agreement (R007). @@ -838,6 +885,10 @@ def _strict_evidence_failure(entry: object, paths: LoopPaths, if not verified["ok"]: return f"is not verified evidence: {[issue['message'] for issue in verified['issues']]}" + verdict = _verdict_failure(record, paths) + if verdict is not None: + return verdict + if bound is not None: committed = bound.get(entry) if committed is None: diff --git a/loop/evidence.py b/loop/evidence.py index f66d3df..292c92b 100644 --- a/loop/evidence.py +++ b/loop/evidence.py @@ -33,10 +33,25 @@ EVIDENCE_SCHEMA_ID = "loop-engineer/evidence@1" +VERIFY_BUNDLE_KIND = "verify-bundle" _URI_PATTERN = re.compile(r"^(?!/)(?![A-Za-z][A-Za-z0-9+.\-]*://).+$") _SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +def verify_bundle_is_green(bundle: Mapping[str, Any]) -> bool: + """The repo's ONE green-marker rule for a verify bundle. + + A bundle is green when it says so explicitly — ``outcome == "PASS"`` or + ``passed is True``. A bundle carrying only a numeric ``score`` reads RED: a + score is not a verdict, and treating one as a pass is exactly the false + completion this kernel exists to refuse (the rule originated in + ``scripts/metrics.py``, which now imports it rather than restating it). + """ + if not isinstance(bundle, Mapping): + return False + return str(bundle.get("outcome", "")).upper() == "PASS" or bundle.get("passed") is True + + class EvidenceError(ValueError): """The workspace-root precondition for evidence verification was not met.""" diff --git a/scripts/metrics.py b/scripts/metrics.py index 84dd3fb..a8c6423 100644 --- a/scripts/metrics.py +++ b/scripts/metrics.py @@ -82,6 +82,7 @@ if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) +from loop.evidence import verify_bundle_is_green # noqa: E402 from loop.paths import LoopPaths, resolve_loop_paths # noqa: E402 import re # noqa: E402 @@ -247,8 +248,8 @@ def _load_verify_bundles(loop_dir: Path) -> list[dict]: data = _read_json_object(path) if not data: continue - outcome = str(data.get("outcome", "")).upper() - green = outcome == "PASS" or data.get("passed") is True + # One definition, shared with the kernel's strict-completion bar. + green = verify_bundle_is_green(data) it = data.get("iteration_id", data.get("iteration")) task = data.get("task") verifier = data.get("verifier") diff --git a/scripts/test_completion_verdict.py b/scripts/test_completion_verdict.py new file mode 100644 index 0000000..792d4c4 --- /dev/null +++ b/scripts/test_completion_verdict.py @@ -0,0 +1,237 @@ +"""The strict mode must cite evidence of PASSING, not merely authentic evidence. + +Every other sub-check of ``all_required_verified_evidence`` asks whether the record is +real: it validates, its pointer resolves and hashes, some event bound those bytes, its +goalpost is live. None of them asked what the verdict SAYS — so a dispatch whose +verifier FAILED produced a perfectly authentic, chain-bound record that backed +``Succeeded`` end to end: ``emit.terminate`` accepted it, ``loop doctor`` reported +``ok: true`` with zero issues, and the reducer replayed ``Succeeded``. + +The verdict is judged by the repo's ONE green-marker rule +(``loop.evidence.verify_bundle_is_green``), which ``scripts/metrics.py`` now imports +rather than restating: a bundle is green when ``outcome == "PASS"`` or +``passed is True``, and a ``score``-only bundle reads RED. A bundle that reads RED to +the metrics gate can therefore never read GREEN to the completion gate. +""" +from __future__ import annotations + +import hashlib +import json + +import metrics +import pytest + +from loop import emit, evidence +from loop.completion import VERIFIED_EVIDENCE_MODE +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.reducer import reduce_events +from loop.runner import dispatch_once + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RECORD = ".loop/evidence/evidence-iter1.json" + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _task(task_id="T-1"): + return {"id": task_id, "title": task_id, "status": "pending", "criterion_ref": task_id, + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None} + + +def _dispatched(tmp_path, name, *, exit_code): + """One real dispatch whose verifier exits with ``exit_code``. + + A failing dispatch still writes and BINDS its evidence — that is the point: the + record is genuine provenance for a verification that did not pass. + """ + workspace = tmp_path / name + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": [_task()]}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": name}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + dispatch_once(workspace) + return workspace + + +def _rewrite_bundle(workspace, text): + """Replace the bundle bytes AND the record digest, so only the verdict is wrong. + + Without re-declaring the digest the record would fail hash verification and the + refusal would be right for the wrong reason. + """ + bundle = workspace / ".loop" / "artifacts" / "verify-iter1.json" + bundle.write_text(text, encoding="utf-8") + record_path = workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + record["sha256"] = hashlib.sha256(text.encode("utf-8")).hexdigest() + record_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return record_path + + +def _rebind(workspace, record_path): + """Re-bind the rewritten record at its new digest, alone, so the ambiguous-binding + refusal (the OTHER new check) cannot be what fires.""" + events = SQLiteEventStore(workspace / ".loop" / "events.db").read("run-1") + for suffix in ("", "-wal", "-shm"): + (workspace / ".loop" / ("events.db" + suffix)).unlink(missing_ok=True) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + digest = hashlib.sha256(record_path.read_bytes()).hexdigest() + bundle = workspace / ".loop" / "artifacts" / "verify-iter1.json" + bundle_digest = hashlib.sha256(bundle.read_bytes()).hexdigest() + for event in events: + hashes = event["artifact_hashes"] + if hashes: + hashes = [{"path": _RECORD, "sha256": digest}, + {"path": ".loop/artifacts/verify-iter1.json", "sha256": bundle_digest}] + store.append("run-1", event["type"], event["payload"], actor=event["actor"], + event_id=event["event_id"], causation_id=event["causation_id"], + correlation_id=event["correlation_id"], ts=event["ts"], + artifact_hashes=hashes or None) + + +def _terminal_file(workspace, entry): + """A hand-written strict terminal — the only way one citing a red verdict reaches + disk now that ``emit.terminate`` refuses to write it.""" + (workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "project": workspace.name, "state": "Succeeded", + "criteria_met": {"T-1": True}, "completion_policy": {"mode": VERIFIED_EVIDENCE_MODE}, + "evidence": [entry], "false_completion": False, + "terminated_at": "2026-07-25T00:00:00+00:00", "reason": "hand-written"}, + indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +# --- the reproduction --------------------------------------------------------- + + +def test_a_failing_verdict_cannot_back_succeeded_at_write_time(tmp_path): + workspace = _dispatched(tmp_path, "failing", exit_code=1) + bundle = json.loads((workspace / ".loop" / "artifacts" / "verify-iter1.json") + .read_text(encoding="utf-8")) + assert bundle["outcome"] == "FAIL" and bundle["passed"] is False # genuine red + with pytest.raises(emit.EmitError, match="verdict is not a pass"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_doctor_reports_a_strict_terminal_backed_by_a_failing_verdict(tmp_path): + workspace = _dispatched(tmp_path, "failing", exit_code=1) + _terminal_file(workspace, _RECORD) + report = doctor_report(workspace) + assert "unverified_evidence_terminal" in _codes(report) + assert any("verdict is not a pass" in issue["message"] for issue in report["issues"]) + + +def test_a_passing_dispatch_is_still_accepted(tmp_path): + """Positive control: the check narrows nothing that was honestly green.""" + workspace = _dispatched(tmp_path, "passing", exit_code=0) + path = emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} + + +# --- the green-marker rule, one definition ------------------------------------ + + +def test_metrics_and_the_completion_bar_share_one_green_marker_definition(): + assert metrics.verify_bundle_is_green is evidence.verify_bundle_is_green + + +@pytest.mark.parametrize("bundle,expected", [ + ({"outcome": "PASS"}, True), + ({"passed": True}, True), + ({"outcome": "pass"}, True), + ({"outcome": "FAIL", "passed": False}, False), + ({"score": 1.0}, False), # a score is not a verdict + ({"passed": 1}, False), # truthy is not True + ({}, False), +]) +def test_the_green_marker_rule(bundle, expected): + assert evidence.verify_bundle_is_green(bundle) is expected + + +def test_a_score_only_bundle_reads_red_at_the_strict_bar(tmp_path): + """The durable repo lesson, now load-bearing in two places at once: a bundle with a + perfect score and no verdict token is RED to metrics, and must be RED here too.""" + workspace = _dispatched(tmp_path, "scored", exit_code=0) + record_path = _rewrite_bundle(workspace, json.dumps({"score": 1.0}) + "\n") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="verdict is not a pass"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +# --- the non-verify-bundle kind, decided explicitly --------------------------- + + +def test_a_record_of_another_kind_cannot_back_succeeded(tmp_path): + """evidence@1's ``kind`` is an open vocabulary (log, diff, screenshot, report). + + A non-``verify-bundle`` record carries no verdict this layer can read, so it is + REFUSED rather than waved through: the strict mode's claim is that completion is + backed by a verification that passed, and an artifact with no verdict cannot make + that claim. + """ + workspace = _dispatched(tmp_path, "logkind", exit_code=0) + record_path = workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + record["kind"] = "log" + record_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="carries no verdict"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_an_unparseable_verdict_is_a_refusal_not_a_skip(tmp_path): + workspace = _dispatched(tmp_path, "garbage", exit_code=0) + record_path = _rewrite_bundle(workspace, "not json at all\n") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="not UTF-8 JSON"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_a_non_object_verdict_is_a_refusal(tmp_path): + workspace = _dispatched(tmp_path, "listy", exit_code=0) + record_path = _rewrite_bundle(workspace, "[]\n") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="not a JSON object"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +# --- the pure layers are unchanged, deliberately ------------------------------ + + +def test_the_reducer_still_admits_a_record_shaped_entry_without_reading_it(): + """Decision 14 stands: the reducer holds no filesystem, so it checks SHAPE only. + + Pinned so the verdict check is not later mistaken for something the pure fold + enforces — a fold that pretended to read a verdict would be the overclaim this + slice exists to prevent. + """ + projection = reduce_events([ + {"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 0, "event_id": "e0", + "type": "contract_opened", "actor": "op", "ts": "2026-07-25T00:00:00+00:00", + "causation_id": None, "correlation_id": None, "artifact_hashes": [], + "payload": {"workspace": "ws"}}, + {"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 1, "event_id": "e1", + "type": "terminal_written", "actor": "op", "ts": "2026-07-25T00:00:01+00:00", + "causation_id": None, "correlation_id": None, "artifact_hashes": [], + "payload": {"state": "Succeeded", "criteria_met": {"T-1": True}, "evidence": [_RECORD], + "false_completion": False, + "completion_policy": {"mode": VERIFIED_EVIDENCE_MODE}}}]) + assert projection["terminal"]["state"] == "Succeeded" From 9608906eda72075a8b2e5f08d6bb787581524849 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 19:26:07 -0400 Subject: [PATCH 16/22] fix(binding): make the write-time bound view conflict-aware and contain the walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects lived in the gap between the write-time and read-time views of "bound". An APPEND-ONLY forge laundered tampered bytes past emit, with no trigger drop and no re-chain: bound_artifact_digests built a {path: sha256} comprehension over every event, so a path bound twice collapsed LAST-WINS. Rewrite a bound record (terminate correctly refuses — "bound at a different digest"), then append ONE ordinary event binding that path at the NEW digest, and the writer accepted the forged tree while _bound_evidence_issues — which checks PER EVENT — still reported evidence_chain_mismatch on it. That layer disagreement is what ruling R9 called the defect, and §17 claims the writer can never accept a terminal its own replay would refuse. The view now returns every DISTINCT digest a path was bound at, in first-bound order (the ordinary repeat-binding case stays a one-element tuple), and the strict bar refuses a path carrying more than one: ambiguous is not proof. Both layers now refuse the same trees. bound_artifact_digests shipped with no direct test at all (Task-3 carry-forward); it has one now. The binding walk also read whatever a path declared. event@1 constrains it only to a non-empty string, so an appended event could make loop doctor open and hash /etc/hostname; the ledger's justification — that the chain hash covers artifact_hashes — is wrong, because the chain prevents retroactive edits to a binding, not a fresh append naming an arbitrary path. Every declared path is now containment-checked with ZERO filesystem access before anything is opened (absolute, drive-lettered, backslashed and ..-traversing paths are reported as the new bound_evidence_escape and never read), symlinked escapes are caught after resolution exactly as verify_evidence catches them, and the read itself is bounded: non-regular files are refused rather than streamed forever, and the hash runs in chunks under a 64 MiB cap. --- loop/contract.py | 22 +- loop/evidence.py | 84 ++++++- loop/runtime.py | 51 +++- reference/repo-os-contract.md | 9 +- scripts/test_adversarial_evidence_binding.py | 4 +- scripts/test_bound_artifact_view.py | 233 +++++++++++++++++++ 6 files changed, 379 insertions(+), 24 deletions(-) create mode 100644 scripts/test_bound_artifact_view.py diff --git a/loop/contract.py b/loop/contract.py index 22adace..4c94338 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -837,7 +837,7 @@ def _verdict_failure(record: Mapping[str, Any], paths: LoopPaths) -> str | None: def _strict_evidence_failure(entry: object, paths: LoopPaths, - bound: Mapping[str, str] | None) -> str | None: + bound: Mapping[str, tuple[str, ...]] | None) -> str | None: """The ONE definition of the verified-evidence bar (plan decision 14). Returns ``None`` when the cited entry can back ``Succeeded``, otherwise a detail @@ -847,9 +847,11 @@ def _strict_evidence_failure(entry: object, paths: LoopPaths, resolves inside the workspace and hashes to the digest it declares; 2. verdict — the artifact it points at is a verify bundle that says PASS. An authentic record of a FAILING verification is still not proof of success; - 3. chain-boundness — some event bound this record's path AT its current bytes. - ``bound is None`` means there is no event store at all, and the sub-check is - skipped: the store-less writer-API path is a DOCUMENTED degradation, not a pass; + 3. chain-boundness — some event bound this record's path AT its current bytes, + and at exactly ONE digest. A path bound at two or more different digests is + ambiguous, and ambiguous is not proof. ``bound is None`` means there is no + event store at all, and the sub-check is skipped: the store-less writer-API + path is a DOCUMENTED degradation, not a pass; 4. goalpost agreement — when the record names a task still in TASKS.json, its recorded ``policy_digest`` equals the live one. A goalpost that cannot even be computed is a FAILURE, not a skip: unestablished is not agreement (R007). @@ -891,12 +893,18 @@ def _strict_evidence_failure(entry: object, paths: LoopPaths, if bound is not None: committed = bound.get(entry) - if committed is None: + if not committed: return ("is not bound into the event chain — no event committed this record's " "digest, so no dispatch produced it") + if len(committed) > 1: + # Append-only forgery: a later ordinary event re-binds a tampered path at its + # new digest. Collapsing the bindings would let the writer accept a tree the + # doctor walk still reports, so an ambiguous binding is refused outright. + return (f"is bound at {len(committed)} different digests " + f"({', '.join(committed)}) — an ambiguous binding is not proof") current = hashlib.sha256(blob).hexdigest() - if committed != current: - return (f"is bound at a different digest: the chain committed {committed}, " + if committed[0] != current: + return (f"is bound at a different digest: the chain committed {committed[0]}, " f"the record now hashes to {current}") produced_by = record.get("produced_by") diff --git a/loop/evidence.py b/loop/evidence.py index 292c92b..a767000 100644 --- a/loop/evidence.py +++ b/loop/evidence.py @@ -25,7 +25,7 @@ import os import re import stat -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Mapping from .contract import ContractIssue, _resolve_requested_mode, _schemas_dir @@ -242,3 +242,85 @@ def verify_evidence(evidence: Mapping[str, Any], *, workspace_root: str | Path) def artifact_object_path(workspace_root: str | Path, sha256: str) -> Path: """Return evidence@1's content-addressed artifact location without I/O.""" return Path(workspace_root) / ".loop" / "artifacts" / "objects" / sha256[:2] / sha256 + + +#: A chain-bound path is attacker-nameable — any event may declare any string. A gate +#: must therefore never perform an unbounded read on one. 64 MiB is far above any +#: artifact this kernel writes and far below "read whatever is at the other end". +MAX_BOUND_ARTIFACT_BYTES = 64 * 1024 * 1024 + +_DRIVE_PREFIX = re.compile(r"^[A-Za-z]:") + + +def _lexical_escape(rel: object) -> str | None: + """Why this declared path is not workspace-relative — decided with ZERO I/O. + + Runs before anything is opened, so an escaping path is reported rather than read. + """ + if not isinstance(rel, str) or not rel.strip(): + return "is not a non-empty path" + if "\\" in rel: + return "contains a backslash, which is not a workspace-relative POSIX path" + if _DRIVE_PREFIX.match(rel): + return "names a drive letter" + pure = PurePosixPath(rel) + if pure.is_absolute(): + return "is an absolute path" + if ".." in pure.parts: + return "traverses out of the workspace with '..'" + return None + + +def hash_bound_artifact( + workspace_root: str | Path, rel: object, *, max_bytes: int = MAX_BOUND_ARTIFACT_BYTES, +) -> tuple[str, str]: + """Containment-check a chain-bound path, then stream-hash it under a cap. + + Returns ``(code, detail)``: + + * ``("ok", )``; + * ``("escape", )`` — the path is not inside the workspace. For a lexical + escape NOTHING on disk was touched; a symlinked escape is caught after + resolution, exactly as ``verify_evidence`` catches it; + * ``("unreadable", )`` — contained, but its bytes could not be hashed: + absent, not a regular file (so ``/dev/zero`` and friends are refused rather + than read forever), or larger than ``max_bytes``. + """ + escape = _lexical_escape(rel) + if escape is not None: + return "escape", escape + root = Path(workspace_root) + try: + resolved = (root / str(rel)).resolve(strict=True) + except (OSError, ValueError, RuntimeError): + return "unreadable", "is absent or cannot be resolved" + try: + resolved.relative_to(root.resolve()) + except ValueError: + return "escape", "resolves outside the workspace (a symlinked component)" + try: + fd = os.open(resolved, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0)) + except OSError: + return "unreadable", "is absent or cannot be opened" + try: + file_stat = os.fstat(fd) + if not stat.S_ISREG(file_stat.st_mode): + return "unreadable", "is not a regular file" + if file_stat.st_size > max_bytes: + return "unreadable", (f"is {file_stat.st_size} bytes, above the {max_bytes}-byte " + f"bound-artifact read cap, so its digest was not computed") + digest = hashlib.sha256() + read = 0 + with os.fdopen(os.dup(fd), "rb") as source: + while chunk := source.read(64 * 1024): + read += len(chunk) + if read > max_bytes: + return "unreadable", (f"grew past the {max_bytes}-byte bound-artifact " + f"read cap while being hashed") + digest.update(chunk) + except OSError: + return "unreadable", "is absent or cannot be read" + finally: + os.close(fd) + return "ok", digest.hexdigest() diff --git a/loop/runtime.py b/loop/runtime.py index 8e300ab..20aa31d 100644 --- a/loop/runtime.py +++ b/loop/runtime.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib import json import shutil import sqlite3 @@ -340,22 +339,35 @@ def _bound_evidence_issues(target: str | Path, mode: str | None) -> list[dict[st Driven by what each event DECLARES: a legacy event carries an empty artifact_hashes list and is silent by construction, because the append-only triggers make a retroactive binding impossible (repo-os-contract.md #22). + + A declared path is attacker-nameable — event@1 constrains it only to a non-empty + string, and the chain covers a binding rather than vouching for it. Every path is + therefore containment-checked BEFORE anything is opened and hashed under a cap + (``loop.evidence.hash_bound_artifact``); an escaping path is reported as + ``bound_evidence_escape``, never read. """ + from .evidence import hash_bound_artifact + _, _run_id, events, _validation = _events(target, mode) workspace = resolve_loop_paths(target).workspace issues: list[dict[str, Any]] = [] for event in events: for entry in event.get("artifact_hashes") or []: - path = workspace / entry["path"] - try: - blob = path.read_bytes() - except OSError: + code, detail = hash_bound_artifact(workspace, entry["path"]) + if code == "escape": + issues.append(ContractIssue( + "bound_evidence_escape", + f"event {event['event_id']} (sequence {event['sequence']}) bound " + f"{entry['path']!r}, which {detail} — a bound path outside the " + f"workspace is a finding, not something to read")) + continue + if code == "unreadable": issues.append(ContractIssue( "missing_bound_evidence", f"event {event['event_id']} (sequence {event['sequence']}) bound " - f"{entry['path']} into the chain but it is absent or unreadable")) + f"{entry['path']} into the chain but it {detail}")) continue - actual = hashlib.sha256(blob).hexdigest() + actual = detail if actual != entry["sha256"]: issues.append(ContractIssue( "evidence_chain_mismatch", @@ -366,8 +378,20 @@ def _bound_evidence_issues(target: str | Path, mode: str | None) -> list[dict[st return issues -def bound_artifact_digests(target: str | Path, mode: str | None = None) -> dict[str, str] | None: - """{workspace-relative POSIX path: sha256} for every artifact any event bound. +def bound_artifact_digests(target: str | Path, + mode: str | None = None) -> dict[str, tuple[str, ...]] | None: + """{workspace-relative POSIX path: every DISTINCT sha256 an event bound it at}. + + Conflict-aware by construction. A dict-comprehension keyed on path would collapse + repeat bindings LAST-WINS, which is not a summary but a laundering channel: an + append-only forge that re-binds a tampered path at its new digest would look bound + to this write-time view while ``_bound_evidence_issues`` — which checks PER EVENT — + still reports ``evidence_chain_mismatch`` on the same tree. Returning the full + conflict set keeps the two views in agreement on every tree; the strict bar refuses + a path carrying more than one digest, because ambiguous is not proof. + + Digests are in first-bound order and de-duplicated, so the ordinary case (the same + path bound repeatedly at the same bytes) stays a one-element tuple. None means there is no event store, and the caller MUST degrade explicitly and say so (decision 14) rather than treat absence as satisfaction. An empty dict means a @@ -377,5 +401,10 @@ def bound_artifact_digests(target: str | Path, mode: str | None = None) -> dict[ if not (resolve_loop_paths(target).loop_dir / "events.db").is_file(): return None _, _run_id, events, _validation = _events(target, mode) - return {entry["path"]: entry["sha256"] - for event in events for entry in event.get("artifact_hashes") or []} + digests: dict[str, list[str]] = {} + for event in events: + for entry in event.get("artifact_hashes") or []: + seen = digests.setdefault(entry["path"], []) + if entry["sha256"] not in seen: + seen.append(entry["sha256"]) + return {path: tuple(seen) for path, seen in digests.items()} diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index c7ae28d..b9b5148 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -1303,11 +1303,11 @@ only finding is the anchor. A present, readable store adds "state_json_agrees", "deterministic", "legal_sequence", "chain"}`; any of `state_field_mismatch`, `desynced_terminal_window`, `terminal_state_mismatch`, `illegal_event_sequence`, `event_chain_broken`, `chain_columns_missing`, -`evidence_chain_mismatch`, `missing_bound_evidence`, or +`evidence_chain_mismatch`, `missing_bound_evidence`, `bound_evidence_escape`, or `chain_anchor_mismatch` fails doctor (`ok: false`). The findings that come from the composed verbs keep the identical issue code those verbs already use; -`chain_columns_missing`, `evidence_chain_mismatch` and `missing_bound_evidence` -are doctor's own store-gated checks, appended to the same list. A store that cannot be read at +`chain_columns_missing`, `evidence_chain_mismatch`, `missing_bound_evidence` and +`bound_evidence_escape` are doctor's own store-gated checks, appended to the same list. A store that cannot be read at all — `corrupt_store`, `empty_store`, `invalid_event`, or `ambiguous_run_id` — also fails doctor rather than being silently skipped; `"event_store"` reports `{"present": true, "readable": false, "error_code": }` in that case. @@ -1340,7 +1340,8 @@ it is the honest statement of how much of the log the chain does not cover. | `self_verified_evidence` | A discovered evidence@1 record declares `produced_by.executor == verified_by.by` (strip+casefold) — the producer verified its own work. Enforces the independence rule of `reference/safety-and-approvals.md` §5, which was prose-only before v0.11.0. | | `missing_evidence_record` | A runner-written verify bundle `.loop/artifacts/verify-iter.json` exists with no matching `.loop/evidence/evidence-iter.json`. Residue of a removed provenance record, in the same family as `missing_event_store`. Fires only when a bundle is present, so an absent-everything contract stays byte-identical. | | `evidence_chain_mismatch` | An artifact whose digest an event bound into the hash chain no longer matches its bytes on disk. The message names the digest the chain committed and the digest found; the original bytes may still remain in the content-addressed object store at `.loop/artifacts/objects//`. | -| `missing_bound_evidence` | An event bound an artifact path into the chain and that path is now absent or unreadable — a deleted bundle/record pair, a deleted object, or the sanctioned crash window between the durable append and the evidence write. Fires only for events that bound something: a legacy event binds nothing and is silent by construction. | +| `missing_bound_evidence` | An event bound an artifact path into the chain and that path is now absent or unreadable — a deleted bundle/record pair, a deleted object, the sanctioned crash window between the durable append and the evidence write, or a path that resolves to something the gate refuses to read (a non-regular file such as a device or FIFO, or a file above the 64 MiB bound-artifact read cap). Fires only for events that bound something: a legacy event binds nothing and is silent by construction. | +| `bound_evidence_escape` | An event bound a path that is not inside the workspace — absolute, drive-lettered, backslashed, `..`-traversing, or resolving outside through a symlinked component. `event@1` constrains `path` only to a non-empty string and the chain covers a binding rather than vouching for it, so the binding walk containment-checks every declared path **before** opening anything: an escaping path is reported, never read. | | `policy_digest_mismatch` | The **latest** evidence record for a task records a `policy_digest` that is not the live `TASKS.json` goalpost — the declared `verify`/`criterion_ref`/`depends_on`/`id` changed after the verification. Also fires when the live entry cannot be canonicalized at all, because a comparison that cannot run has not passed. Re-verify to record the current goalpost. | | `unverified_evidence_terminal` | A `Succeeded` terminal declaring `completion_policy.mode: all_required_verified_evidence` has an evidence entry that fails one of the mode's three checks: it is not a hash-verified evidence@1 record; or (when an event store exists) no event bound those bytes into the chain; or the record's `policy_digest` is not the live `TASKS.json` goalpost for the task it names. The message names which. An unreadable store is also a failure. In a contract with **no** event store the middle check is skipped — the mode's strength is store-dependent by construction (§17). | diff --git a/scripts/test_adversarial_evidence_binding.py b/scripts/test_adversarial_evidence_binding.py index c925f8a..4feeb78 100644 --- a/scripts/test_adversarial_evidence_binding.py +++ b/scripts/test_adversarial_evidence_binding.py @@ -253,7 +253,9 @@ def test_a_full_rewrite_of_artifacts_and_store_is_not_caught_without_an_anchor_p new_sha = _rewrite_everything(workspace) # the forge is real, not a no-op: different bytes, and the chain now commits to them assert new_sha != before - assert bound_artifact_digests(workspace)[".loop/artifacts/verify-iter1.json"] == new_sha + # One digest, not a conflict set: the re-chain REPLACES the binding rather than + # appending a second one, which is exactly why this rewrite still verifies clean. + assert bound_artifact_digests(workspace)[".loop/artifacts/verify-iter1.json"] == (new_sha,) assert json.loads(_bundle(workspace).read_text(encoding="utf-8"))["forged"] is True report = doctor_report(workspace) assert report["ok"] is True and report["issues"] == [] diff --git a/scripts/test_bound_artifact_view.py b/scripts/test_bound_artifact_view.py new file mode 100644 index 0000000..6e33bc5 --- /dev/null +++ b/scripts/test_bound_artifact_view.py @@ -0,0 +1,233 @@ +"""The write-time and read-time views of "bound" must agree on every tree. + +Two defects lived in the gap between them. + +**The append-only forge.** ``bound_artifact_digests`` built a ``{path: sha256}`` dict +comprehension over every event, so a path bound twice collapsed LAST-WINS. No trigger +drop and no re-chain were needed: rewrite a bound record (``emit.terminate`` correctly +refuses — "bound at a different digest"), then append ONE ordinary event binding that +path at the NEW digest, and the writer's view said "bound, and it matches" while +``_bound_evidence_issues`` — which checks PER EVENT — still reported +``evidence_chain_mismatch`` on the same tree. A path carrying two or more different +digests is ambiguous, and ambiguous is not proof, so the view now returns the whole +conflict set and the strict bar refuses it. + +**The unbounded read of an attacker-named path.** ``event@1`` constrains a bound +``path`` only to ``minLength: 1``, and the walk joined it straight onto the workspace, +so an appended event could make ``loop doctor`` open and hash ``/etc/hostname``. Every +other workspace-bearing layer containment-checks; this one now does too, BEFORE it +opens anything, and reads under a cap. + +``bound_artifact_digests`` shipped with no direct test at all (Task-3 carry-forward); +it has one now. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from loop import emit +from loop.completion import VERIFIED_EVIDENCE_MODE +from loop.contract import doctor_report +from loop.evidence import hash_bound_artifact +from loop.events import SQLiteEventStore +from loop.runner import dispatch_once +from loop.runtime import RuntimeStoreError, bound_artifact_digests +from loop.scaffold import scaffold + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RUN_ID = "run-1" +_RECORD = ".loop/evidence/evidence-iter1.json" + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _store(workspace): + return SQLiteEventStore(workspace / ".loop" / "events.db") + + +def _dispatched(tmp_path, name="workspace"): + workspace = tmp_path / name + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [{"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "T-1", + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None}], + }), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = _store(workspace) + store.append(_RUN_ID, "contract_opened", {"workspace": name}, actor="test") + for state in _RAMP: + store.append(_RUN_ID, "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + dispatch_once(workspace) + return workspace + + +def _append_binding(workspace, artifact_hashes, *, iteration_id=2): + _store(workspace).append( + _RUN_ID, "iteration_appended", + {"iteration_id": iteration_id, "outcome": "task_passed", "state": "execute-task"}, + actor="forge", artifact_hashes=artifact_hashes) + + +# --- the append-only forge ---------------------------------------------------- + + +def _forge(workspace): + """Rewrite the bound record, then re-bind it at its new digest by APPENDING.""" + record_path = workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + record["verified_by"]["by"] = "forged-ci" + text = json.dumps(record, indent=2, sort_keys=True) + "\n" + record_path.write_text(text, encoding="utf-8") + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def test_an_append_only_forge_cannot_launder_a_tampered_record(tmp_path): + workspace = _dispatched(tmp_path) + digest = _forge(workspace) + # Control: before the re-bind BOTH layers already refuse, for the same reason. + with pytest.raises(emit.EmitError, match="bound at a different digest"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + assert "evidence_chain_mismatch" in _codes(doctor_report(workspace)) + + _append_binding(workspace, [{"path": _RECORD, "sha256": digest}]) + # The read-time walk still reports the FIRST binding's mismatch... + assert "evidence_chain_mismatch" in _codes(doctor_report(workspace)) + # ...and the write-time view no longer disagrees with it. + with pytest.raises(emit.EmitError, match="ambiguous binding is not proof"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +# --- bound_artifact_digests, directly ----------------------------------------- + + +def test_no_store_returns_none_and_nothing_bound_returns_an_empty_dict(tmp_path): + scaffolded = tmp_path / "scaffolded" + scaffold(scaffolded) + assert bound_artifact_digests(scaffolded) is None + bare = tmp_path / "bare" + emit.open_contract(bare) + _store(bare).append(_RUN_ID, "contract_opened", {"workspace": "bare"}, actor="test") + assert bound_artifact_digests(bare) == {} + + +def test_a_dispatch_binds_three_paths_each_at_exactly_one_digest(tmp_path): + workspace = _dispatched(tmp_path) + bound = bound_artifact_digests(workspace) + assert set(bound) == {".loop/artifacts/verify-iter1.json", _RECORD, + *(p for p in bound if p.startswith(".loop/artifacts/objects/"))} + assert all(len(digests) == 1 for digests in bound.values()) + + +def test_repeat_bindings_at_the_same_digest_stay_one_element(tmp_path): + workspace = _dispatched(tmp_path) + same = bound_artifact_digests(workspace)[_RECORD][0] + _append_binding(workspace, [{"path": _RECORD, "sha256": same}]) + assert bound_artifact_digests(workspace)[_RECORD] == (same,) + + +def test_conflicting_bindings_are_returned_in_first_bound_order(tmp_path): + workspace = _dispatched(tmp_path) + first = bound_artifact_digests(workspace)[_RECORD][0] + second, third = "a" * 64, "b" * 64 + _append_binding(workspace, [{"path": _RECORD, "sha256": second}], iteration_id=2) + _append_binding(workspace, [{"path": _RECORD, "sha256": third}], iteration_id=3) + assert bound_artifact_digests(workspace)[_RECORD] == (first, second, third) + + +def test_an_unreadable_store_raises_rather_than_returning_an_empty_view(tmp_path): + workspace = _dispatched(tmp_path) + for sidecar in ("events.db-wal", "events.db-shm"): + (workspace / ".loop" / sidecar).unlink(missing_ok=True) + (workspace / ".loop" / "events.db").write_bytes(b"not a sqlite database" * 64) + with pytest.raises(RuntimeStoreError): + bound_artifact_digests(workspace) + + +# --- containment in the binding walk ------------------------------------------ + + +@pytest.mark.parametrize("escaping", [ + "/etc/hostname", + "../../../../etc/hostname", + "nested/../../outside.txt", + "C:/Windows/system.ini", + "..\\..\\etc\\hostname", + " ", +]) +def test_an_escaping_bound_path_is_reported_not_read(tmp_path, escaping): + workspace = _dispatched(tmp_path) + _append_binding(workspace, [{"path": escaping, "sha256": "0" * 64}]) + report = doctor_report(workspace) + assert "bound_evidence_escape" in _codes(report) + assert report["ok"] is False + + +def test_the_lexical_containment_check_touches_no_filesystem(tmp_path, monkeypatch): + """Proof, not assertion: with ``os.open`` and ``Path.resolve`` booby-trapped, an + escaping path still returns a verdict — so nothing was opened or even stat'd.""" + def boom(*args, **kwargs): # pragma: no cover - must never run + raise AssertionError("the walk touched the filesystem before containment") + + monkeypatch.setattr("loop.evidence.os.open", boom) + monkeypatch.setattr(Path, "resolve", boom) + code, detail = hash_bound_artifact(tmp_path, "../../../../etc/hostname") + assert code == "escape" and ".." in detail + + +@pytest.mark.parametrize("rel", ["", " ", None, 7]) +def test_a_path_that_is_not_a_path_is_an_escape_not_a_crash(tmp_path, rel): + """``event@1``'s ``minLength: 1`` keeps the empty string out of a real store, so the + helper's own total-ness is what stops a hand-built or future store from crashing the + walk instead of reporting it.""" + code, detail = hash_bound_artifact(tmp_path, rel) + assert code == "escape" and "non-empty path" in detail + + +def test_a_symlinked_escape_is_caught_after_resolution(tmp_path): + workspace = tmp_path / "ws" + workspace.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("secret", encoding="utf-8") + (workspace / "link.txt").symlink_to(outside) + code, detail = hash_bound_artifact(workspace, "link.txt") + assert code == "escape" and "outside the workspace" in detail + + +def test_a_contained_regular_file_hashes(tmp_path): + (tmp_path / "a.txt").write_text("hello", encoding="utf-8") + assert hash_bound_artifact(tmp_path, "a.txt") == ( + "ok", hashlib.sha256(b"hello").hexdigest()) + + +def test_a_non_regular_file_is_unreadable_rather_than_read_forever(tmp_path): + os.mkfifo(tmp_path / "pipe") + code, detail = hash_bound_artifact(tmp_path, "pipe") + assert code == "unreadable" and "not a regular file" in detail + + +def test_the_read_is_capped(tmp_path): + (tmp_path / "big.bin").write_bytes(b"x" * 4096) + code, detail = hash_bound_artifact(tmp_path, "big.bin", max_bytes=16) + assert code == "unreadable" and "read cap" in detail + + +def test_an_absent_bound_path_is_still_missing_bound_evidence(tmp_path): + """The pre-existing code is unchanged for the pre-existing case.""" + workspace = _dispatched(tmp_path) + (workspace / ".loop" / "evidence" / "evidence-iter1.json").unlink() + assert "missing_bound_evidence" in _codes(doctor_report(workspace)) From 74d368b8a7dd6378dab662c9cc81364a525606d8 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 19:26:10 -0400 Subject: [PATCH 17/22] fix(doctor): rank unnumbered evidence records instead of skipping them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The goalpost sweep dropped every record outside the runner's evidence-iter.json name, on the reasoning that such a record carries no position in the run's order and so can never be shown to be latest. That created a hole: a record with no iteration id is still the LATEST when it is the only one for its task. A store-less contract whose single hand-written record was named evidence-hand.json stayed doctor-clean with zero issues after its goalpost moved, while the identical record renamed evidence-iter9.json fired policy_digest_mismatch. §17 and §22 meanwhile both claim "the latest evidence record for a task" is compared. Latest now RANKS unnumbered records below every numbered one, ties broken by filename — the same deterministic tie-break the numbered path already used — so the only record for a task is always compared. Decision 5 is untouched: still only the latest record per task, and a record naming a task absent from TASKS.json is still never compared. --- loop/contract.py | 25 +++++--- scripts/test_doctor_evidence_verification.py | 64 +++++++++++++++++++- 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/loop/contract.py b/loop/contract.py index 4c94338..7dc17f7 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -697,6 +697,11 @@ def _policy_digest_issues( Only the latest record backs the task's current claim; older records describe goalposts that were current when they were written, and comparing them would make every honest re-verification a permanent failure (repo-os-contract.md §17). + "Latest" orders numbered ``evidence-iter.json`` records by iteration and ranks + every unnumbered record below all of them (ties by filename), so a task whose only + record carries no iteration id is still compared — being unnumbered is not a way + out of the comparison. + A record naming a task that is no longer in TASKS.json is not compared at all — a renamed or removed task is a replan, not a forgery. A task that is present but cannot be canonicalized is reported rather than skipped: a comparison that cannot @@ -708,20 +713,22 @@ def _policy_digest_issues( declared = tasks.get("tasks") if isinstance(tasks, dict) else None entries = {t["id"]: t for t in (declared if isinstance(declared, list) else []) if isinstance(t, dict) and isinstance(t.get("id"), str)} - latest: dict[str, tuple[int, Path, dict]] = {} + latest: dict[str, tuple[tuple[int, int, str], Path, dict]] = {} for iteration_id, record_path, data in records: - if not isinstance(iteration_id, int): - # A record outside the runner's evidence-iter.json name carries no - # position in the run's order, so it can never be shown to be latest. - continue produced_by = data.get("produced_by") task_id = produced_by.get("task_id") if isinstance(produced_by, dict) else None if not isinstance(task_id, str) or task_id not in entries: continue - if task_id not in latest or (iteration_id, record_path.name) > ( - latest[task_id][0], latest[task_id][1].name): - latest[task_id] = (iteration_id, record_path, data) - for task_id, (_iteration_id, record_path, data) in sorted(latest.items()): + # A record outside the runner's evidence-iter.json name carries no position + # in the run's order, so it sorts BELOW every numbered record (ties by + # filename). Excluding it outright left a hole: the only record for a task is + # still that task's latest, and skipping it meant a moved goalpost went + # unreported for exactly the hand-written records most worth comparing. + rank = ((1, iteration_id, record_path.name) if isinstance(iteration_id, int) + else (0, 0, record_path.name)) + if task_id not in latest or rank > latest[task_id][0]: + latest[task_id] = (rank, record_path, data) + for task_id, (_rank, record_path, data) in sorted(latest.items()): verified_by = data.get("verified_by") recorded = verified_by.get("policy_digest") if isinstance(verified_by, dict) else None if not isinstance(recorded, str): diff --git a/scripts/test_doctor_evidence_verification.py b/scripts/test_doctor_evidence_verification.py index 5a159f9..ebe992f 100644 --- a/scripts/test_doctor_evidence_verification.py +++ b/scripts/test_doctor_evidence_verification.py @@ -29,7 +29,8 @@ def _codes(report): "verify": "./scripts/verify-fast.sh", "depends_on": [], "attempts": 0, "evidence": None} -def _ws(tmp_path, *, records, bundle_text='{"outcome": "PASS", "passed": true}', tasks=(_TASK,)): +def _ws(tmp_path, *, records=(), named_records=(), + bundle_text='{"outcome": "PASS", "passed": true}', tasks=(_TASK,)): target = tmp_path / "workspace" scaffold(target) (target / "TASKS.json").write_text(json.dumps( @@ -44,6 +45,8 @@ def _ws(tmp_path, *, records, bundle_text='{"outcome": "PASS", "passed": true}', for iteration_id, record in records: (directory / f"evidence-iter{iteration_id}.json").write_text( json.dumps(record), encoding="utf-8") + for name, record in named_records: + (directory / name).write_text(json.dumps(record), encoding="utf-8") return target @@ -106,6 +109,65 @@ def test_only_the_latest_record_per_task_is_compared(tmp_path): assert "policy_digest_mismatch" not in _codes(doctor_report(target)) +# --- unnumbered records are ranked, never excluded ---------------------------- +# +# The sweep used to `continue` on any record outside `evidence-iter.json`, so a +# task whose ONLY record carried no iteration id was never compared: move its +# goalpost and doctor stayed ok with zero issues, while the identical record renamed +# `evidence-iter9.json` fired. Being unnumbered was a way out of the comparison, and +# §22 meanwhile claimed "the latest evidence record for a task" is compared. Latest +# now RANKS the unnumbered below every numbered record (ties by filename) rather +# than dropping them, so the only record for a task is always compared. + + +def test_a_tasks_only_record_is_compared_even_when_it_is_unnumbered(tmp_path): + moved = dict(_TASK, verify="true") + target = _ws(tmp_path, named_records=[("evidence-hand.json", _record())], tasks=(moved,)) + assert "policy_digest_mismatch" in _codes(doctor_report(target)) + + +def test_an_unnumbered_record_is_silent_when_its_goalpost_agrees(tmp_path): + """Positive control: ranking them in did not make every hand-written record a finding. + + Scoped to the goalpost code — the fixture's numbered BUNDLE legitimately reports + ``missing_evidence_record`` (no ``evidence-iter1.json`` describes it), which is a + different check and not what this test is about. + """ + target = _ws(tmp_path, named_records=[("evidence-hand.json", _record())]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_a_numbered_record_outranks_an_unnumbered_one_for_the_same_task(tmp_path): + """An unnumbered record carries no position in the run's order, so it can never be + the latest while a numbered record exists — the stale digest below is not compared.""" + target = _ws(tmp_path, + records=[(2, _record(iteration_id=2))], + named_records=[("evidence-hand.json", _record(policy_digest="d" * 64))]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_unnumbered_records_tie_break_by_filename(tmp_path): + """With no ordering to appeal to, filename order is the deterministic tie-break — + the same rule the numbered path already used for equal iteration ids.""" + target = _ws(tmp_path, named_records=[ + ("evidence-a.json", _record(policy_digest="d" * 64)), + ("evidence-b.json", _record()), + ]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + later_is_stale = _ws(tmp_path / "second", named_records=[ + ("evidence-a.json", _record()), + ("evidence-b.json", _record(policy_digest="d" * 64)), + ]) + assert "policy_digest_mismatch" in _codes(doctor_report(later_is_stale)) + + +def test_an_unnumbered_record_naming_an_unknown_task_is_still_not_compared(tmp_path): + """Decision 5 is untouched: a renamed or removed task is a replan, not a forgery.""" + target = _ws(tmp_path, named_records=[ + ("evidence-hand.json", _record(task_id="T-404", policy_digest="d" * 64))]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + def test_a_record_naming_an_unknown_task_is_not_compared(tmp_path): target = _ws(tmp_path, records=[(1, _record(task_id="T-404", policy_digest="d" * 64))]) assert "policy_digest_mismatch" not in _codes(doctor_report(target)) From 7b0b26591766320dd4262db0397b5c856e0fa2e6 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 19:26:13 -0400 Subject: [PATCH 18/22] =?UTF-8?q?test(conformance):=20scope=20the=20=C2=A7?= =?UTF-8?q?22=20doc=20pin=20to=20the=20=C2=A722=20table=20so=20it=20can=20?= =?UTF-8?q?fail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_every_new_doctor_code_is_documented_in_section_22 grepped the WHOLE contract document, and every code it names also appears in §17's tier list — so deleting all four §22 rows left the pin green. Verified: with the rows removed, a whole-file backtick search still hits all four. This is the pin a PR body would cite as evidence that the codes are documented, and it documented nothing. It now slices §22's New issue codes table the way the sibling §17 pin already slices its section, and covers bound_evidence_escape as well. Proven to discriminate by deleting the five rows and observing the failure. --- scripts/test_conformance.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/test_conformance.py b/scripts/test_conformance.py index 33c8826..90a7380 100644 --- a/scripts/test_conformance.py +++ b/scripts/test_conformance.py @@ -500,11 +500,28 @@ def test_documented_artifact_binding_vector_matches_the_writer(tmp_path): f"§17 lists the bound set out of writer order: {positions}") +def _new_issue_codes_table() -> str: + """§22's ``**New issue codes.**`` table, heading row to the end of the table.""" + text = _CONTRACT_DOC.read_text(encoding="utf-8") + marker = "**New issue codes.**" + assert marker in text, "§22 does not publish a new-issue-codes table" + start = text.index(marker) + end = text.find("\n\n", text.index("|---|---|", start)) + return text[start:] if end == -1 else text[start:end] + + def test_every_new_doctor_code_is_documented_in_section_22(): - doc = _CONTRACT_DOC.read_text(encoding="utf-8") + """Scoped to the §22 TABLE, not the whole file. + + Grepping the whole document could not fail: every one of these codes also appears + in §17's tier list, so deleting all four §22 rows left the pin green — the pin a PR + body would cite as proof that the codes are documented. + """ + table = _new_issue_codes_table() for code in ("evidence_chain_mismatch", "missing_bound_evidence", - "policy_digest_mismatch", "unverified_evidence_terminal"): - assert f"`{code}`" in doc, f"§22 does not document {code}" + "policy_digest_mismatch", "unverified_evidence_terminal", + "bound_evidence_escape"): + assert f"`{code}`" in table, f"§22's new-issue-codes table does not document {code}" def test_no_shipped_surface_still_claims_evidence_is_unverified(): From c0e41daeb450e128ac718616a1dd9062befb9972 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 19:26:16 -0400 Subject: [PATCH 19/22] fix(doctor,emit): guard the third store read and the object-collision read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two typed-error escapes. doctor's chain-binding walk is a THIRD independent read of the event store, and it sat OUTSIDE the except RuntimeStoreError that its two siblings (status_report, replay_report) share. A store that became unreadable between reads escaped doctor_report as an untyped traceback instead of the typed finding an errored check owes (R007). write_verify_evidence read the existing object from inside its except FileExistsError handler, where the sibling except OSError cannot reach — so an IsADirectoryError or PermissionError at the content-addressed object path escaped the writer untyped. The collision read is now wrapped, and the pre-existing typed refusal for a genuine digest collision is unchanged. --- loop/emit.py | 10 ++- loop/runtime.py | 6 +- scripts/test_doctor_store_read_guard.py | 55 +++++++++++++++ .../test_verify_evidence_object_collision.py | 67 +++++++++++++++++++ 4 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 scripts/test_doctor_store_read_guard.py create mode 100644 scripts/test_verify_evidence_object_collision.py diff --git a/loop/emit.py b/loop/emit.py index 9318ee2..cbadda3 100644 --- a/loop/emit.py +++ b/loop/emit.py @@ -568,7 +568,15 @@ def write_verify_evidence( built.object_path.parent.mkdir(parents=True, exist_ok=True) _atomic_create_text(built.object_path, built.bundle_text) except FileExistsError: - existing = hashlib.sha256(built.object_path.read_bytes()).hexdigest() + # The sibling `except OSError` cannot catch what is raised INSIDE this handler, + # so the collision read is wrapped here: an IsADirectoryError or PermissionError + # at the object path must leave write_verify_evidence as a typed EmitError. + try: + existing = hashlib.sha256(built.object_path.read_bytes()).hexdigest() + except OSError as exc: + raise EmitError( + f"object store entry for {built.sha256} exists but cannot be read at " + f"{built.object_path}: {exc}") from exc if existing != built.sha256: raise EmitError( f"object store holds different bytes for {built.sha256}: refusing to overwrite " diff --git a/loop/runtime.py b/loop/runtime.py index 20aa31d..71ab11f 100644 --- a/loop/runtime.py +++ b/loop/runtime.py @@ -302,6 +302,10 @@ def event_consistency_issues( status = status_report(target, mode=mode) replay = replay_report(target, mode=mode) declares_chain_without_columns = _store_declares_chain_without_columns(path) + # Inside the guard with its two siblings: this is a THIRD independent read, and a + # store that becomes unreadable between reads must surface as a typed finding + # rather than an untyped traceback out of doctor_report (R007). + bound_issues = _bound_evidence_issues(target, mode) except RuntimeStoreError as exc: unreadable_issues = [ContractIssue(exc.code, str(exc))] if expect_chain_head is not None: @@ -309,7 +313,7 @@ def event_consistency_issues( "an anchored chain head was supplied but the event store cannot be read")) return {"present": True, "readable": False, "error_code": exc.code}, unreadable_issues issues = list(status["divergence"]) + list(replay["findings"]) - issues.extend(_bound_evidence_issues(target, mode)) + issues.extend(bound_issues) if declares_chain_without_columns: issues.append(ContractIssue( "chain_columns_missing", diff --git a/scripts/test_doctor_store_read_guard.py b/scripts/test_doctor_store_read_guard.py new file mode 100644 index 0000000..e48c325 --- /dev/null +++ b/scripts/test_doctor_store_read_guard.py @@ -0,0 +1,55 @@ +"""Doctor's THIRD store read is guarded exactly like its two siblings (R007). + +``event_consistency_issues`` reads the store three times: ``status_report``, +``replay_report``, and the chain-binding walk. The first two sat inside an +``except RuntimeStoreError`` that turns an unreadable store into a typed finding; the +walk sat OUTSIDE it, so a store that became unreadable between reads escaped +``doctor_report`` as an untyped traceback — an errored check must fail, never explode. +""" +from __future__ import annotations + +import json + +from loop import emit +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.runner import dispatch_once +from loop.runtime import RuntimeStoreError + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + + +def _dispatched(tmp_path): + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [{"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "T-1", + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None}], + }), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + dispatch_once(workspace) + return workspace + + +def test_a_store_that_dies_between_reads_is_a_typed_finding_not_a_traceback( + tmp_path, monkeypatch): + workspace = _dispatched(tmp_path) + + def boom(*args, **kwargs): + raise RuntimeStoreError("corrupt_store", "the store died between reads") + + monkeypatch.setattr("loop.runtime._bound_evidence_issues", boom) + report = doctor_report(workspace) # must not raise + assert "corrupt_store" in {issue["code"] for issue in report["issues"]} + assert report["event_store"] == {"present": True, "readable": False, + "error_code": "corrupt_store"} diff --git a/scripts/test_verify_evidence_object_collision.py b/scripts/test_verify_evidence_object_collision.py new file mode 100644 index 0000000..99b4ac5 --- /dev/null +++ b/scripts/test_verify_evidence_object_collision.py @@ -0,0 +1,67 @@ +"""The object-store collision handler must not raise a raw OSError. + +``write_verify_evidence`` places the content-addressed object with a create-once hard +link and treats an existing object as idempotent success once its bytes match. The +match check reads the existing object from INSIDE the ``except FileExistsError`` +handler, where the sibling ``except OSError`` cannot reach it — so an +``IsADirectoryError`` or ``PermissionError`` at the object path escaped the writer +untyped, past the API's typed-error contract. +""" +from __future__ import annotations + +import json + +import pytest + +from loop import emit +from loop.evidence import artifact_object_path +from loop.verifier import injected_verifier_identity + +_TASK = {"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "C-1", + "verify": "./scripts/verify-fast.sh", "depends_on": [], "attempts": 0, "evidence": None} + + +def _contract(tmp_path, name="workspace"): + workspace = tmp_path / name + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": [_TASK]}), encoding="utf-8") + return workspace + + +def _built(workspace): + return emit.build_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + + +def test_an_unreadable_existing_object_is_a_typed_emit_error(tmp_path): + workspace = _contract(tmp_path) + built = _built(workspace) + # A DIRECTORY at the object path: the hard link refuses it (FileExistsError) and the + # collision read then raises IsADirectoryError from inside the handler. + built.object_path.mkdir(parents=True) + with pytest.raises(emit.EmitError, match="cannot be read"): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + + +def test_a_colliding_object_with_different_bytes_is_still_the_refusal_it_was(tmp_path): + """Control: the pre-existing typed refusal for a genuine digest collision is unchanged.""" + workspace = _contract(tmp_path) + built = _built(workspace) + built.object_path.parent.mkdir(parents=True, exist_ok=True) + built.object_path.write_text("different bytes\n", encoding="utf-8") + with pytest.raises(emit.EmitError, match="refusing to overwrite"): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + + +def test_an_identical_existing_object_is_idempotent_success(tmp_path): + """Control: a re-run of the same dispatch still writes cleanly.""" + workspace = _contract(tmp_path) + first = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + second = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + assert first["sha256"] == second["sha256"] + assert artifact_object_path(workspace, first["sha256"]).is_file() From 1c4e70c781f95740dc09e97efb324ba1dbd0ac45 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 19:26:18 -0400 Subject: [PATCH 20/22] docs(reference): name the four-part strict bar and three behavioural changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §17 now describes the bar this branch actually ships — four sub-checks, not three: the verdict check and its one green-marker rule (a score-only bundle reads red in both the FCR gate and the completion gate; a non-verify-bundle kind carries no verdict and is refused), and why an ambiguous binding is refused rather than collapsed. The latest-per-task rule is restated to say that unnumbered records are ranked in, not dropped. New "Upgrade notes" block, stated without hedging because each one can bite an existing user: - policy_digest_mismatch is NOT opt-in. A contract clean at v0.10.0 goes red on upgrade after an ordinary goalpost edit, and the documented remedy — re-verify — does not generally help, because done-ness comes from the event log so the runner never re-verifies. Measured, the reliable route back to green is renaming the task, which is the very evasion the tier list admits the check cannot catch. The operator's real options are named, and the absent re-baseline affordance is named as absent. - The completion_policy enum widening is forward-incompatible as a HARD ERROR: an older kernel reports invalid_completion_policy + schema_violation rather than degrading, which matters for the README's pinned @v0.10.0 action and for uvx loop-engineer@0.10.0. - os.link now runs once PER DISPATCH rather than once per run, so a filesystem without hard-link support turns every dispatch into a committed-then-failed iteration. Named as a known limitation; there is deliberately no non-atomic fallback. --- reference/repo-os-contract.md | 84 +++++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index b9b5148..b28c2c6 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -1053,7 +1053,7 @@ which declare no policy at all — keeps the old bar forever The two workspace-bearing layers share **one predicate** (`loop.contract._strict_evidence_failure`); `emit.terminate` imports it rather -than restating it, because two hand-written copies of a three-part check drift +than restating it, because two hand-written copies of a four-part check drift and a drift here is a silent false completion. That predicate is a strict **superset** of the pure half: it applies the reducer's own `evidence_entry_is_record_shaped` to the cited entry first, so the writer can @@ -1064,12 +1064,34 @@ check shape and nothing else because they hold no filesystem — making them **What the mode proves, exactly.** For every cited entry: (1) the entry resolves to a readable evidence@1 record that validates, and the artifact its `uri` names resolves inside the workspace and hashes to the digest the record declares; -(2) **when an event store exists**, those record bytes are the bytes some event -bound into the chain, at that digest; and (3) when the record names a task still -in `TASKS.json`, its recorded `policy_digest` equals the live goalpost. A goalpost -that cannot even be **computed** is a failure, not a skip — unestablished is not -agreement. An event store that cannot be **read** is likewise a refusal, never a -pass. The message names which of the three sub-checks failed. +(2) that artifact is a **verify bundle whose verdict is a pass**; (3) **when an +event store exists**, those record bytes are the bytes some event bound into the +chain, at that digest, and at exactly **one** digest; and (4) when the record +names a task still in `TASKS.json`, its recorded `policy_digest` equals the live +goalpost. A goalpost that cannot even be **computed** is a failure, not a skip — +unestablished is not agreement. An event store that cannot be **read** is likewise +a refusal, never a pass. The message names which of the four sub-checks failed. + +**Check (2), the verdict, in full.** Authenticity is not success. A dispatch whose +verifier FAILS still writes and binds a perfectly genuine record — of a failure — +so the bar asks what the artifact SAYS as well as whether it is real. Green is the +repo's one green-marker rule, `loop.evidence.verify_bundle_is_green`: `outcome == +"PASS"` **or** `passed is true`. A bundle carrying only a numeric `score` reads +**red**, here and in `scripts/metrics.py`, which imports the same predicate — a +bundle that reads red to the FCR gate can never read green to the completion gate. +`kind` is an open vocabulary, and a record that is **not** a `verify-bundle` (a +log, a diff, a screenshot) carries no verdict this layer can read: it is refused, +because an artifact with no verdict cannot show that anything passed. An +unreadable or unparseable verdict is a refusal, never a skip. + +**Check (3), and why ambiguity is refused.** The write-time view +(`loop.runtime.bound_artifact_digests`) reports every **distinct** digest a path +was bound at, not the last one. Collapsing repeat bindings would be a laundering +channel rather than a summary: an append-only forge that re-binds a tampered path +at its new digest would look bound to the writer while the per-event walk still +reported `evidence_chain_mismatch` on the same tree. A path bound at two or more +different digests is therefore refused outright, which is what keeps the writer +from ever accepting a terminal its own replay would refuse. **What it does not prove, stated without hedging.** It does not prove that a human, a runner, or anything at all *produced* the record: a hand-written record @@ -1095,7 +1117,10 @@ anchor, deleting the store silently disables this mode's chain-boundness check. **A deliberate asymmetry with doctor's general goalpost check.** Doctor compares the **latest record per task** (see the tier list below); the terminal check -compares **every cited record**. Citing a record in a `Succeeded` terminal is a +compares **every cited record**. "Latest" orders numbered +`evidence-iter.json` records by iteration and ranks every **unnumbered** record +below all of them, ties broken by filename — unnumbered records are ranked in, not +dropped, so a task whose only record carries no iteration id is still compared. Citing a record in a `Succeeded` terminal is a present-tense claim that *this* record backs completion now, so a stale goalpost in a cited record is a false completion, not an artifact of history. @@ -1107,6 +1132,49 @@ in `reason` — either `tasks with no evidence record: …` or `evidence records not meet the verified-evidence bar: `. Downgrading silently would be a self-serving choice; both branches are pinned. +### Upgrade notes — three behavioural changes, stated without hedging + +These are not opt-in, and two of them can turn a contract that was clean on the +previous release red on this one. They are recorded here rather than left to be +discovered. + +**1. `policy_digest_mismatch` is NOT opt-in, and its remedy is currently +unsatisfying.** The verified-evidence *completion mode* is opt-in; this doctor +check is not. A contract that was doctor-clean at v0.10.0 goes **red** on upgrade +as soon as anyone makes an ordinary goalpost edit — changing a task's `verify`, +`criterion_ref`, `depends_on` or `id` after a verification was recorded. The +documented remedy is "re-verify to record the current goalpost", and in a +loop whose done-ness comes from the event log the runner will not re-verify a task +it already considers done, so re-verification does not generally help. Measured, +the reliable route back to green is to **rename the task** — which is precisely the +evasion the tier list below admits this check cannot catch (a record naming a task +absent from `TASKS.json` is never compared). So the check is honest about a moved +goalpost and weak against a determined one, and the operator's real options today +are: edit goalposts *before* the verification that records them, re-run the +dispatch so a fresh record is written, or accept the finding as the true statement +that the declared goalpost moved after the work was verified. A first-class +"re-baseline this task's goalpost" affordance does not exist yet. + +**2. The `completion_policy` enum widening is forward-INCOMPATIBLE as a hard +error.** `all_required_verified_evidence` is a new enum member, so an **older** +kernel reading a terminal that declares it reports `invalid_completion_policy` +and `schema_violation` — it does not degrade to the old bar, it fails. This +matters concretely for the README's pinned `@v0.10.0` action and for +`uvx loop-engineer@0.10.0`: a gate pinned to the older release will reject a +contract written by the newer one. Pin the gate and the writer to the same +release, or keep writing `all_required` until the gate is upgraded. + +**3. `os.link` now runs once PER DISPATCH, not once per run.** The +content-addressed object is placed with the same create-once hard link as the +immutable terminal record, and every dispatch places one. On a filesystem without +hard-link support (some network and container-overlay mounts, and any FAT-family +volume) the link fails, so **every dispatch becomes a committed-then-failed +iteration**: the durable event is already appended when the object write raises. +Previously the same limitation existed but was hit once, at terminal write. There +is no fallback copy path by design — a non-atomic create would reopen the +overwrite race the hard link closes — so this is a known limitation of running a +loop on such a filesystem, not a configuration option. + ### The integrity boundary, in four honest tiers Not a single "surfaces / does not surface" pair: From 31d0adab695404c254bb8d536618f5d9e011c7e2 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 19:26:20 -0400 Subject: [PATCH 21/22] docs(reference): the unverified_evidence_terminal row names four checks, not three --- reference/repo-os-contract.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index b28c2c6..6a1c05b 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -1411,7 +1411,7 @@ it is the honest statement of how much of the log the chain does not cover. | `missing_bound_evidence` | An event bound an artifact path into the chain and that path is now absent or unreadable — a deleted bundle/record pair, a deleted object, the sanctioned crash window between the durable append and the evidence write, or a path that resolves to something the gate refuses to read (a non-regular file such as a device or FIFO, or a file above the 64 MiB bound-artifact read cap). Fires only for events that bound something: a legacy event binds nothing and is silent by construction. | | `bound_evidence_escape` | An event bound a path that is not inside the workspace — absolute, drive-lettered, backslashed, `..`-traversing, or resolving outside through a symlinked component. `event@1` constrains `path` only to a non-empty string and the chain covers a binding rather than vouching for it, so the binding walk containment-checks every declared path **before** opening anything: an escaping path is reported, never read. | | `policy_digest_mismatch` | The **latest** evidence record for a task records a `policy_digest` that is not the live `TASKS.json` goalpost — the declared `verify`/`criterion_ref`/`depends_on`/`id` changed after the verification. Also fires when the live entry cannot be canonicalized at all, because a comparison that cannot run has not passed. Re-verify to record the current goalpost. | -| `unverified_evidence_terminal` | A `Succeeded` terminal declaring `completion_policy.mode: all_required_verified_evidence` has an evidence entry that fails one of the mode's three checks: it is not a hash-verified evidence@1 record; or (when an event store exists) no event bound those bytes into the chain; or the record's `policy_digest` is not the live `TASKS.json` goalpost for the task it names. The message names which. An unreadable store is also a failure. In a contract with **no** event store the middle check is skipped — the mode's strength is store-dependent by construction (§17). | +| `unverified_evidence_terminal` | A `Succeeded` terminal declaring `completion_policy.mode: all_required_verified_evidence` has an evidence entry that fails one of the mode's four checks: it is not a hash-verified evidence@1 record; or it does not point at a `verify-bundle` whose verdict is a pass; or (when an event store exists) no event bound those bytes into the chain, or events bound that path at two or more different digests; or the record's `policy_digest` is not the live `TASKS.json` goalpost for the task it names. The message names which. An unreadable store is also a failure. In a contract with **no** event store the chain check is skipped — the mode's strength is store-dependent by construction (§17). | **Do not confuse `missing_evidence_record` with `missing_evidence_path`.** They are opposite directions of the same pair. `missing_evidence_record` walks From 42c093173265ab2567487eb62433daba4f9ea748 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sat, 25 Jul 2026 20:22:29 -0400 Subject: [PATCH 22/22] fix(evidence): narrow the binding invariant to what it proves, and close four edges The Check (3) paragraph asserted that refusing an ambiguous binding "keeps the writer from ever accepting a terminal its own replay would refuse" -- unconditional, and false. The write-time bar examines the entries a terminal cites and the artifact each uri names; it does not walk the other paths those events bound. The two layers agree about every path a terminal rests on, which is the claim; they are not two spellings of the same sweep. Also: pin that the canonical green-marker OR rule accepts a self-contradicting {outcome: PASS, passed: false} bundle, since tightening it here alone would make one bundle green to the completion bar and red to the FCR gate that shares the object; record the 64 MiB bound-artifact read cap as a fourth un-remediable upgrade note; stop claiming a malformed bound path is "outside the workspace" when it is not a path at all; and close an fd leak when os.fdopen raises after os.dup succeeds. --- loop/evidence.py | 8 +++++++- loop/runtime.py | 4 ++-- reference/repo-os-contract.md | 22 +++++++++++++++++++--- scripts/test_completion_verdict.py | 21 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/loop/evidence.py b/loop/evidence.py index a767000..b56f61d 100644 --- a/loop/evidence.py +++ b/loop/evidence.py @@ -312,7 +312,13 @@ def hash_bound_artifact( f"bound-artifact read cap, so its digest was not computed") digest = hashlib.sha256() read = 0 - with os.fdopen(os.dup(fd), "rb") as source: + duplicate = os.dup(fd) + try: + source = os.fdopen(duplicate, "rb") + except OSError: + os.close(duplicate) # fdopen never took ownership + raise + with source: while chunk := source.read(64 * 1024): read += len(chunk) if read > max_bytes: diff --git a/loop/runtime.py b/loop/runtime.py index 71ab11f..7c994e7 100644 --- a/loop/runtime.py +++ b/loop/runtime.py @@ -362,8 +362,8 @@ def _bound_evidence_issues(target: str | Path, mode: str | None) -> list[dict[st issues.append(ContractIssue( "bound_evidence_escape", f"event {event['event_id']} (sequence {event['sequence']}) bound " - f"{entry['path']!r}, which {detail} — a bound path outside the " - f"workspace is a finding, not something to read")) + f"{entry['path']!r}, which {detail} — a bound path that does not " + f"resolve inside the workspace is a finding, not something to read")) continue if code == "unreadable": issues.append(ContractIssue( diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index 6a1c05b..b818e83 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -1090,8 +1090,15 @@ was bound at, not the last one. Collapsing repeat bindings would be a laundering channel rather than a summary: an append-only forge that re-binds a tampered path at its new digest would look bound to the writer while the per-event walk still reported `evidence_chain_mismatch` on the same tree. A path bound at two or more -different digests is therefore refused outright, which is what keeps the writer -from ever accepting a terminal its own replay would refuse. +different digests is therefore refused outright. + +The scope of that refusal is the **cited** evidence, and no wider. The write-time +bar examines each entry a `Succeeded` terminal cites and the artifact that entry's +`uri` names; it does not walk the other paths those events happened to bind. So a +tree can carry an ambiguously-bound artifact that no terminal cites, and the +writer will not see it while `loop doctor`'s per-event walk still reports +`evidence_chain_mismatch`. The two layers agree about every path a terminal +rests on, which is the claim; they are not two spellings of the same sweep. **What it does not prove, stated without hedging.** It does not prove that a human, a runner, or anything at all *produced* the record: a hand-written record @@ -1132,7 +1139,7 @@ in `reason` — either `tasks with no evidence record: …` or `evidence records not meet the verified-evidence bar: `. Downgrading silently would be a self-serving choice; both branches are pinned. -### Upgrade notes — three behavioural changes, stated without hedging +### Upgrade notes — four behavioural changes, stated without hedging These are not opt-in, and two of them can turn a contract that was clean on the previous release red on this one. They are recorded here rather than left to be @@ -1175,6 +1182,15 @@ is no fallback copy path by design — a non-atomic create would reopen the overwrite race the hard link closes — so this is a known limitation of running a loop on such a filesystem, not a configuration option. +**4. A bound artifact above the read cap fails doctor, and there is no knob.** The +binding walk refuses to hash a bound path larger than `MAX_BOUND_ARTIFACT_BYTES` +(64 MiB) and reports `missing_bound_evidence`, because a gate must not perform an +unbounded read on a path an event names. A loop that legitimately binds an +artifact above that size — a large log, a coverage dump — therefore goes red with +no in-product remedy and no configuration option, the same un-satisfiable shape as +note 1. Bind a digest of the large artifact rather than the artifact itself, or +keep it out of `artifact_hashes`. + ### The integrity boundary, in four honest tiers Not a single "surfaces / does not surface" pair: diff --git a/scripts/test_completion_verdict.py b/scripts/test_completion_verdict.py index 792d4c4..773307d 100644 --- a/scripts/test_completion_verdict.py +++ b/scripts/test_completion_verdict.py @@ -173,6 +173,27 @@ def test_a_score_only_bundle_reads_red_at_the_strict_bar(tmp_path): evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) +def test_a_self_contradicting_verdict_reads_green_pinned(tmp_path): + """The OR rule's cost, pinned rather than discovered later. + + ``verify_bundle_is_green`` is satisfied by EITHER token, so a bundle claiming + ``outcome: PASS`` while ``passed`` is false reads green and can back a strict + ``Succeeded``. That is the repo's canonical rule, shared object-for-object with + the FCR gate, so tightening it here alone would make one bundle green to the + completion bar and red to metrics — a worse defect than the one it fixes. + Changing it is a repo-wide decision about what a verdict means, not a patch. + """ + workspace = _dispatched(tmp_path, "contradicting", exit_code=0) + record_path = _rewrite_bundle( + workspace, json.dumps({"outcome": "PASS", "passed": False}) + "\n") + _rebind(workspace, record_path) + path = emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} # accepted + assert evidence.verify_bundle_is_green({"outcome": "PASS", "passed": False}) is True + + # --- the non-verify-bundle kind, decided explicitly ---------------------------