diff --git a/README.md b/README.md index f72ecee..51100ca 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,10 @@ ships, on disk and runnable today: named reason otherwise — e.g. `python3 -m pytest`), policy digest — and `loop doctor` fails when a record declares that its own producer verified it; identity is recorded, not proven: a worker can write a - false verifier name, and the records live outside the hash chain. + false verifier name. A verified dispatch **binds its evidence digests into the + hash chain**, and `loop doctor` re-hashes them — binding makes tampering + detectable against an anchor, not impossible: without `--expect-chain-head` a + worker who can rewrite `.loop/` can rewrite the chain too. ![The inspector scores a self-asserted DIY loop 0/weak, then the gate-backed example 90/strong — both runs live](docs/demo.gif) diff --git a/loop/completion.py b/loop/completion.py index 6d9a1ba..62fcca7 100644 --- a/loop/completion.py +++ b/loop/completion.py @@ -11,9 +11,15 @@ from collections.abc import Mapping from typing import Final, Literal, TypeAlias, cast -CompletionMode: TypeAlias = Literal["all_required"] +CompletionMode: TypeAlias = Literal["all_required", "all_required_verified_evidence"] DEFAULT_COMPLETION_MODE: Final[CompletionMode] = "all_required" -SUPPORTED_COMPLETION_MODES: Final[tuple[CompletionMode, ...]] = (DEFAULT_COMPLETION_MODE,) +VERIFIED_EVIDENCE_MODE: Final[CompletionMode] = "all_required_verified_evidence" +SUPPORTED_COMPLETION_MODES: Final[tuple[CompletionMode, ...]] = ( + DEFAULT_COMPLETION_MODE, + VERIFIED_EVIDENCE_MODE, +) + +_RECORD_PREFIX = ".loop/evidence/" class CompletionPolicyError(ValueError): @@ -63,11 +69,24 @@ def criteria_satisfy_completion( ``True``; truthy substitutes such as ``1`` are deliberately rejected. """ normalized = normalize_completion_policy(policy) - if normalized["mode"] == "all_required": + if normalized["mode"] in SUPPORTED_COMPLETION_MODES: + # The criteria half is deliberately shared: the verified-evidence mode raises + # the EVIDENCE bar, never the criteria bar. return bool(criteria_met) and all(value is True for value in criteria_met.values()) raise AssertionError(f"unhandled completion policy: {normalized!r}") +def policy_requires_verified_evidence(policy: object | None = None) -> bool: + """True when this policy's evidence bar is hash-verified records, not path strings.""" + return normalize_completion_policy(policy)["mode"] == VERIFIED_EVIDENCE_MODE + + +def evidence_entry_is_record_shaped(entry: object) -> bool: + """The structural half of the bar — the most a pure, I/O-free layer can honestly check.""" + return (isinstance(entry, str) and entry.startswith(_RECORD_PREFIX) + and entry.endswith(".json") and ".." not in entry) + + def unmet_required_criteria(criteria_met: Mapping[str, object]) -> tuple[str, ...]: """Return stable string identifiers for criteria not proven true.""" return tuple(sorted(key for key, value in criteria_met.items() if value is not True)) diff --git a/loop/contract.py b/loop/contract.py index 9ec0e44..7dc17f7 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -1,15 +1,20 @@ from __future__ import annotations +import hashlib import json import re from pathlib import Path from typing import Any, Mapping from . import fsm +from .chain import ChainHashError from .completion import ( + VERIFIED_EVIDENCE_MODE, CompletionPolicyError, criteria_satisfy_completion, + evidence_entry_is_record_shaped, normalize_completion_policy, + policy_requires_verified_evidence, unmet_required_criteria, ) from .paths import ARTIFACTS_DIR_NAME, EVIDENCE_DIR_NAME, LoopPaths, resolve_loop_paths @@ -635,6 +640,7 @@ def _validate_jsonl(path: Path, schema_key: str, mode: str, issues: list[dict]) _RUNNER_BUNDLE_RE = re.compile(r"verify-iter([0-9]+)\.json") +_EVIDENCE_RECORD_RE = re.compile(r"evidence-iter([0-9]+)\.json") def _self_verified(record: Mapping[str, Any]) -> bool: @@ -681,36 +687,288 @@ def _orphan_bundle_issues(paths: LoopPaths, issues: list[dict]) -> None: bundle_path)) +def _policy_digest_issues( + paths: LoopPaths, + records: list[tuple[int | None, Path, dict]], + issues: list[dict], +) -> None: + """Compare the LATEST record per task against the live TASKS.json goalpost. + + Only the latest record backs the task's current claim; older records describe + goalposts that were current when they were written, and comparing them would + make every honest re-verification a permanent failure (repo-os-contract.md §17). + "Latest" orders numbered ``evidence-iter.json`` records by iteration and ranks + every unnumbered record below all of them (ties by filename), so a task whose only + record carries no iteration id is still compared — being unnumbered is not a way + out of the comparison. + + A record naming a task that is no longer in TASKS.json is not compared at all — + a renamed or removed task is a replan, not a forgery. A task that is present but + cannot be canonicalized is reported rather than skipped: a comparison that cannot + run has not passed. + """ + from .verifier import verification_policy_digest + + tasks = _read_json(paths.tasks, []) # already reported by the contract read + declared = tasks.get("tasks") if isinstance(tasks, dict) else None + entries = {t["id"]: t for t in (declared if isinstance(declared, list) else []) + if isinstance(t, dict) and isinstance(t.get("id"), str)} + latest: dict[str, tuple[tuple[int, int, str], Path, dict]] = {} + for iteration_id, record_path, data in records: + produced_by = data.get("produced_by") + task_id = produced_by.get("task_id") if isinstance(produced_by, dict) else None + if not isinstance(task_id, str) or task_id not in entries: + continue + # A record outside the runner's evidence-iter.json name carries no position + # in the run's order, so it sorts BELOW every numbered record (ties by + # filename). Excluding it outright left a hole: the only record for a task is + # still that task's latest, and skipping it meant a moved goalpost went + # unreported for exactly the hand-written records most worth comparing. + rank = ((1, iteration_id, record_path.name) if isinstance(iteration_id, int) + else (0, 0, record_path.name)) + if task_id not in latest or rank > latest[task_id][0]: + latest[task_id] = (rank, record_path, data) + for task_id, (_rank, record_path, data) in sorted(latest.items()): + verified_by = data.get("verified_by") + recorded = verified_by.get("policy_digest") if isinstance(verified_by, dict) else None + if not isinstance(recorded, str): + continue + try: + live = verification_policy_digest(entries[task_id]) + except ChainHashError as exc: + # Fail closed. Structural-fallback mode carries no tasks@1 type-check, so + # skipping here would leave a stale goalpost reported by nothing at all. + issues.append(ContractIssue( + "policy_digest_mismatch", + f"{record_path.name}: the goalpost recorded for task {task_id!r} " + f"({recorded}) cannot be compared against the live TASKS.json entry, " + f"because that entry is not canonicalizable ({exc}) — agreement is " + f"unestablished, and unestablished is not agreement", + record_path)) + continue + if live != recorded: + issues.append(ContractIssue( + "policy_digest_mismatch", + f"{record_path.name}: the goalpost recorded for task {task_id!r} " + f"({recorded}) is not the live TASKS.json goalpost ({live}) — " + f"the declared verify/criterion_ref/depends_on/id changed after this " + f"verification; re-verify to record the current goalpost", + record_path)) + + def _validate_evidence_records(paths: LoopPaths, mode: str, issues: list[dict]) -> bool: - """Validate declared evidence@1 records and surface declared self-verification. + """Validate declared evidence@1 records, hash-verify what they reference, and + surface declared self-verification. Declared location: `.loop/evidence/*.json` (repo-os-contract.md §17). An absent directory is a no-op, so a contract with no evidence produces a byte-identical report — the same rule §22 pins for an absent event store. """ - from .evidence import evidence_issues # local: loop.evidence imports this module + from .evidence import evidence_issues, verify_evidence # local: loop.evidence imports this module _orphan_bundle_issues(paths, issues) evidence_dir = paths.loop_dir / EVIDENCE_DIR_NAME if not evidence_dir.is_dir(): return False checked = False + verifiable: list[tuple[int | None, Path, dict]] = [] for record_path in sorted(evidence_dir.glob("*.json")): data = _read_json(record_path, issues) if data is None: continue checked = True - for issue in evidence_issues(data, resolved_mode=mode): + record_issues = evidence_issues(data, resolved_mode=mode) + for issue in record_issues: issues.append(ContractIssue(issue["code"], f"{record_path.name}: {issue['message']}", record_path)) + if not record_issues: + # Guarded: verify_evidence() re-runs validate_evidence internally, so an + # unconditional call would report a malformed record twice. + result = verify_evidence(data, workspace_root=paths.workspace) + for issue in result["issues"]: + issues.append(ContractIssue(issue["code"], f"{record_path.name}: {issue['message']}", record_path)) + match = _EVIDENCE_RECORD_RE.fullmatch(record_path.name) + verifiable.append((int(match.group(1)) if match else None, record_path, data)) if _self_verified(data): issues.append(ContractIssue( "self_verified_evidence", f"{record_path.name}: produced_by.executor == verified_by.by " f"({data['produced_by']['executor']!r}) — the producer declares it verified its own work", record_path)) + _policy_digest_issues(paths, verifiable, issues) return checked +def _verdict_failure(record: Mapping[str, Any], paths: LoopPaths) -> str | None: + """Why the record's artifact does not attest a PASS, or ``None`` when it does. + + Hash-verification proves the pointer resolves to the bytes the record committed + to; it says nothing about what those bytes SAY. A record backing ``Succeeded`` + must point at a green verdict, judged by the repo's one green-marker rule + (``loop.evidence.verify_bundle_is_green``) — the same rule ``scripts/metrics.py`` + scores FCR with, so a bundle that reads RED to metrics can never read GREEN here. + + A record whose ``kind`` is anything other than ``verify-bundle`` carries no + verdict this layer can read (evidence@1's ``kind`` is an open vocabulary — a log, + a diff, a screenshot). Such a record is REFUSED rather than waved through: the + strict mode's claim is that completion is backed by a verification that passed, + and an artifact with no verdict cannot make that claim. Unreadable or + unparseable is likewise a refusal, never a skip (R007). + """ + from .evidence import VERIFY_BUNDLE_KIND, verify_bundle_is_green + + kind = record.get("kind") + if kind != VERIFY_BUNDLE_KIND: + return (f"is a {kind!r} record, not a {VERIFY_BUNDLE_KIND}: it carries no verdict, " + f"so it cannot show that anything passed") + uri = record["uri"] + try: + blob = (paths.workspace / uri).read_bytes() + except (OSError, ValueError) as exc: + return f"cites a verify bundle that cannot be read ({exc})" + # Re-anchored to the digest the record declares, so the bytes judged here are the + # bytes hash-verification just accepted rather than whatever landed since. + if hashlib.sha256(blob).hexdigest() != record["sha256"]: + return (f"cites a verify bundle whose bytes changed between hash verification " + f"and the verdict read: {uri}") + try: + bundle = json.loads(blob.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return f"cites a verify bundle that is not UTF-8 JSON ({exc})" + if not isinstance(bundle, dict): + return "cites a verify bundle that is not a JSON object" + if not verify_bundle_is_green(bundle): + return (f"cites a verify bundle whose verdict is not a pass " + f"(outcome={bundle.get('outcome')!r}, passed={bundle.get('passed')!r}) — " + f"evidence of failure cannot back Succeeded") + return None + + +def _strict_evidence_failure(entry: object, paths: LoopPaths, + bound: Mapping[str, tuple[str, ...]] | None) -> str | None: + """The ONE definition of the verified-evidence bar (plan decision 14). + + Returns ``None`` when the cited entry can back ``Succeeded``, otherwise a detail + string naming which of the four sub-checks failed: + + 1. self-consistency — the entry is a readable evidence@1 record whose ``uri`` + resolves inside the workspace and hashes to the digest it declares; + 2. verdict — the artifact it points at is a verify bundle that says PASS. An + authentic record of a FAILING verification is still not proof of success; + 3. chain-boundness — some event bound this record's path AT its current bytes, + and at exactly ONE digest. A path bound at two or more different digests is + ambiguous, and ambiguous is not proof. ``bound is None`` means there is no + event store at all, and the sub-check is skipped: the store-less writer-API + path is a DOCUMENTED degradation, not a pass; + 4. goalpost agreement — when the record names a task still in TASKS.json, its + recorded ``policy_digest`` equals the live one. A goalpost that cannot even be + computed is a FAILURE, not a skip: unestablished is not agreement (R007). + + ``emit.terminate`` imports this function rather than restating it. Two hand-written + copies of a three-part security check drift, and a drift here is a silent false + completion. + """ + from .evidence import validate_evidence, verify_evidence # local: loop.evidence imports this module + from .verifier import verification_policy_digest + + if not isinstance(entry, str) or not entry.strip(): + return "is not a workspace-relative evidence record path" + if not evidence_entry_is_record_shaped(entry): + # The pure layers reject this shape outright, so accepting it here would let a + # terminal through the writer that its own replay would refuse. + return ("is not verified evidence: not a workspace-relative " + ".loop/evidence/*.json record path") + try: + blob = (paths.workspace / entry).read_bytes() + except (OSError, ValueError) as exc: + return f"is not verified evidence: the record cannot be read ({exc})" + try: + record = json.loads(blob.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return f"is not verified evidence: the record is not UTF-8 JSON ({exc})" + if not isinstance(record, dict): + return "is not verified evidence: the record is not a JSON object" + validation = validate_evidence(record, mode="basic") + if not validation["ok"]: + return f"is not verified evidence: {[issue['message'] for issue in validation['issues']]}" + verified = verify_evidence(record, workspace_root=paths.workspace) + if not verified["ok"]: + return f"is not verified evidence: {[issue['message'] for issue in verified['issues']]}" + + verdict = _verdict_failure(record, paths) + if verdict is not None: + return verdict + + if bound is not None: + committed = bound.get(entry) + if not committed: + return ("is not bound into the event chain — no event committed this record's " + "digest, so no dispatch produced it") + if len(committed) > 1: + # Append-only forgery: a later ordinary event re-binds a tampered path at its + # new digest. Collapsing the bindings would let the writer accept a tree the + # doctor walk still reports, so an ambiguous binding is refused outright. + return (f"is bound at {len(committed)} different digests " + f"({', '.join(committed)}) — an ambiguous binding is not proof") + current = hashlib.sha256(blob).hexdigest() + if committed[0] != current: + return (f"is bound at a different digest: the chain committed {committed[0]}, " + f"the record now hashes to {current}") + + produced_by = record.get("produced_by") + task_id = produced_by.get("task_id") if isinstance(produced_by, dict) else None + if not isinstance(task_id, str): + return None + declared = _read_json(paths.tasks, []) # already reported by the contract read + entries = declared.get("tasks") if isinstance(declared, dict) else None + live = next((task for task in (entries if isinstance(entries, list) else []) + if isinstance(task, dict) and task.get("id") == task_id), None) + if live is None: + # A renamed or removed task is a replan, not a forgery (decision 5). + return None + verified_by = record.get("verified_by") + recorded = verified_by.get("policy_digest") if isinstance(verified_by, dict) else None + try: + current_goalpost = verification_policy_digest(live) + except ChainHashError as exc: + return (f"records a goalpost that cannot be compared against the live TASKS.json " + f"goalpost for task {task_id!r}: that entry is not canonicalizable ({exc}), " + f"so agreement is unestablished — and unestablished is not agreement") + if recorded != current_goalpost: + return (f"records a goalpost that is not the live TASKS.json goalpost for task " + f"{task_id!r}: recorded {recorded!r}, live {current_goalpost!r}") + return None + + +def _check_verified_evidence_terminal(terminal: Any, paths: LoopPaths, issues: list[dict]) -> None: + """Read-time twin of emit.terminate's strict bar — one predicate, two callers.""" + if not isinstance(terminal, dict) or terminal.get("state") != "Succeeded": + return + try: + if not policy_requires_verified_evidence(terminal.get("completion_policy")): + return + except CompletionPolicyError: + return # already reported by _validate_terminal / the terminal@1 schema + from .runtime import RuntimeStoreError, bound_artifact_digests # local: runtime imports this module + + try: + bound = bound_artifact_digests(paths.workspace) + except RuntimeStoreError as exc: + issues.append(ContractIssue( + "unverified_evidence_terminal", + f"the terminal declares {VERIFIED_EVIDENCE_MODE} but the event store could not " + f"be read ({exc}) — chain-boundness is unestablished, and unestablished is not " + f"proof", paths.terminal)) + return + evidence = terminal.get("evidence") + for entry in evidence if isinstance(evidence, list) else []: + detail = _strict_evidence_failure(entry, paths, bound) + if detail is not None: + issues.append(ContractIssue( + "unverified_evidence_terminal", + f"the terminal declares {VERIFIED_EVIDENCE_MODE} but {entry!r} {detail}", + paths.terminal)) + + def _validate_optional_records(paths: LoopPaths, mode: str, issues: list[dict]) -> set[str]: """Validate record files that are present; return the set of record schema keys actually checked (``repair``/``rollout``/``receipt``/``evidence``) so ``doctor`` can @@ -812,6 +1070,8 @@ def validate_contract(target: str | Path, *, mode: str | None = None) -> dict[st _check_stub_verify_scripts(paths, issues) _check_verify_surface(paths, tasks, issues) records_checked = _validate_optional_records(paths, mode, issues) + if terminal is not None: + _check_verified_evidence_terminal(terminal, paths, issues) schemas_checked = list(SCHEMA_IDS) + [ schema_id for key, schema_id in _RECORD_SCHEMA_IDS if key in records_checked diff --git a/loop/emit.py b/loop/emit.py index c85b839..cbadda3 100644 --- a/loop/emit.py +++ b/loop/emit.py @@ -11,6 +11,7 @@ import json import os import tempfile +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping, Sequence @@ -18,18 +19,21 @@ from . import fsm from .chain import ChainHashError from .completion import ( + VERIFIED_EVIDENCE_MODE, CompletionPolicyError, criteria_satisfy_completion, normalize_completion_policy, + policy_requires_verified_evidence, unmet_required_criteria, ) from .contract import ( TERMINAL_STATES, + _strict_evidence_failure, _validate_record, _validate_terminal, _validation_mode, ) -from .evidence import EVIDENCE_SCHEMA_ID +from .evidence import EVIDENCE_SCHEMA_ID, artifact_object_path, validate_evidence from .paths import ARTIFACTS_DIR_NAME, EVIDENCE_DIR_NAME, resolve_loop_paths from .scaffold import scaffold from .verifier import criterion_partition, verification_policy_digest @@ -242,6 +246,28 @@ def append_receipt( return receipts +def _refuse_unverified_evidence(paths, evidence: Sequence[str]) -> None: + """Enforce the verified-evidence bar at write time, through contract's one definition. + + An unreadable store is a REFUSAL, never a skip: the whole point of the mode is that + a cited record is bound into a chain, and a chain nobody can read proves nothing. + """ + from .runtime import RuntimeStoreError, bound_artifact_digests # local: runtime -> events -> emit + + try: + bound = bound_artifact_digests(paths.workspace) + except RuntimeStoreError as exc: + raise EmitError( + f"refusing Succeeded under {VERIFIED_EVIDENCE_MODE}: the event store could not be " + f"read ({exc}) — chain-boundness is unestablished, and unestablished is not proof" + ) from exc + for entry in evidence: + detail = _strict_evidence_failure(entry, paths, bound) + if detail is not None: + raise EmitError( + f"refusing Succeeded under {VERIFIED_EVIDENCE_MODE}: {entry} {detail}") + + def terminate( target: str | Path, *, @@ -302,6 +328,10 @@ def terminate( raise EmitError( "refusing Succeeded because not all required criteria are proven true: " + detail ) + if policy_requires_verified_evidence(normalized_policy): + # Resolved here rather than by hoisting the assignment below, so the order in + # which terminate reports its refusals is exactly what it was. + _refuse_unverified_evidence(_require_contract(target), evidence) paths = _require_contract(target) terminal_path = paths.loop_dir / "terminal_state.json" @@ -407,23 +437,36 @@ def sync_state_to_projection(target: str | Path, projection: dict[str, Any]) -> DEFAULT_VERIFIER_IDENTITY = "loop.run" -def write_verify_evidence( +@dataclass(frozen=True) +class BuiltEvidence: + """Every byte and digest of one verify write, computed without touching disk. + + Separating the computation from the write lets a caller commit to the exact + artifact hashes BEFORE anything is placed, so the digests a durable event + carries and the bytes on disk cannot disagree. + """ + + bundle_path: Path + bundle_text: str + sha256: str + record_path: Path + record_text: str + record_sha256: str + object_path: Path + artifact_hashes: tuple[dict[str, str], ...] + + +def build_verify_evidence( target: str | Path, *, run_id: str, iteration_id: int, task: Mapping[str, Any], passed: bool, code_identity: Mapping[str, Any], summary: str = "", executor: str | None = None, verifier_identity: str | None = None, attempt: int | None = None, -) -> dict[str, Any]: - """Write one verify bundle and its hashed evidence@1 record. - - ``code_identity`` is REQUIRED and is never derived here: only the caller knows - which verifier actually ran (see loop.verifier.executed_verifier_identity vs - injected_verifier_identity). Deriving it from ``task['verify']`` would record a - command that an injected verifier never executed. +) -> BuiltEvidence: + """Render one verify bundle, its evidence@1 record, and their bound digests. - The bundle is the artifact (metrics-compatible; carries verifier identity, the - verdict's source, and the DECLARED criterion partition); the record is the - schema-validated pointer that commits to the bundle's bytes. Identities are - recorded, never inferred: an unsupplied executor is the literal ``unattributed`` - and an unsupplied attempt is ``null``. + Pure with respect to the workspace: it reads the contract and writes nothing. + The record it renders is byte-identical to what ``write_verify_evidence`` + places, and is schema-validated here so an invalid record is refused before + any file exists rather than after. """ paths = _require_contract(target) iteration_id = _require_iteration_id(iteration_id) @@ -450,12 +493,8 @@ def write_verify_evidence( "partition": criterion_partition(task), } bundle_path = paths.loop_dir / ARTIFACTS_DIR_NAME / f"verify-iter{iteration_id}.json" - bundle_path.parent.mkdir(parents=True, exist_ok=True) bundle_text = json.dumps(bundle, indent=2, sort_keys=True) + "\n" - # Staged under a name metrics does not rglob, so a failure before the record is - # written cannot leave an orphan green bundle for FCR to count. - staged = bundle_path.with_name(bundle_path.name + ".staged") - _atomic_write_text(staged, bundle_text) + bundle_sha256 = hashlib.sha256(bundle_text.encode("utf-8")).hexdigest() now = datetime.now(timezone.utc).isoformat(timespec="seconds") record = { @@ -463,7 +502,7 @@ def write_verify_evidence( "id": f"{run_id}:{iteration_id}:verify", "kind": "verify-bundle", "uri": bundle_path.relative_to(paths.workspace).as_posix(), - "sha256": hashlib.sha256(bundle_text.encode("utf-8")).hexdigest(), + "sha256": bundle_sha256, "media_type": "application/json", "created_at": now, "produced_by": {"run_id": run_id, "task_id": task.get("id"), "attempt": attempt, @@ -473,12 +512,90 @@ def write_verify_evidence( "code_digest_basis": identity["code_digest_basis"], "policy_digest": identity["policy_digest"]}, } + validation = validate_evidence(record, mode="basic") + if not validation["ok"]: + raise EmitError(f"evidence record failed schema validation: {validation['issues']}") + record_path = paths.loop_dir / EVIDENCE_DIR_NAME / f"evidence-iter{iteration_id}.json" - record_path.parent.mkdir(parents=True, exist_ok=True) + record_text = json.dumps(record, indent=2, sort_keys=True) + "\n" + record_sha256 = hashlib.sha256(record_text.encode("utf-8")).hexdigest() + object_path = artifact_object_path(paths.workspace, bundle_sha256) + artifact_hashes = tuple( + {"path": path.relative_to(paths.workspace).as_posix(), "sha256": digest} + for path, digest in ((bundle_path, bundle_sha256), (record_path, record_sha256), + (object_path, bundle_sha256)) + ) + return BuiltEvidence( + bundle_path=bundle_path, bundle_text=bundle_text, sha256=bundle_sha256, + record_path=record_path, record_text=record_text, record_sha256=record_sha256, + object_path=object_path, artifact_hashes=artifact_hashes, + ) + + +def write_verify_evidence( + target: str | Path, *, run_id: str, iteration_id: int, task: Mapping[str, Any], passed: bool, + code_identity: Mapping[str, Any], summary: str = "", executor: str | None = None, + verifier_identity: str | None = None, attempt: int | None = None, + built: BuiltEvidence | None = None, +) -> dict[str, Any]: + """Write one verify bundle, its content-addressed object, and its evidence@1 record. + + ``code_identity`` is REQUIRED and is never derived here: only the caller knows + which verifier actually ran (see loop.verifier.executed_verifier_identity vs + injected_verifier_identity). Deriving it from ``task['verify']`` would record a + command that an injected verifier never executed. + + The bundle is the artifact (metrics-compatible; carries verifier identity, the + verdict's source, and the DECLARED criterion partition); the record is the + schema-validated pointer that commits to the bundle's bytes. Identities are + recorded, never inferred: an unsupplied executor is the literal ``unattributed`` + and an unsupplied attempt is ``null``. + + ``built`` lets a caller that already rendered the write place exactly those + bytes, so the digests it committed to elsewhere describe what landed. + """ + if built is None: + built = build_verify_evidence( + target, run_id=run_id, iteration_id=iteration_id, task=task, passed=passed, + code_identity=code_identity, summary=summary, executor=executor, + verifier_identity=verifier_identity, attempt=attempt, + ) + + # The object is placed FIRST and never replaced: it is the recovery copy the + # evidence record's digest still resolves to after the mutable bundle path is + # overwritten. + try: + built.object_path.parent.mkdir(parents=True, exist_ok=True) + _atomic_create_text(built.object_path, built.bundle_text) + except FileExistsError: + # The sibling `except OSError` cannot catch what is raised INSIDE this handler, + # so the collision read is wrapped here: an IsADirectoryError or PermissionError + # at the object path must leave write_verify_evidence as a typed EmitError. + try: + existing = hashlib.sha256(built.object_path.read_bytes()).hexdigest() + except OSError as exc: + raise EmitError( + f"object store entry for {built.sha256} exists but cannot be read at " + f"{built.object_path}: {exc}") from exc + if existing != built.sha256: + raise EmitError( + f"object store holds different bytes for {built.sha256}: refusing to overwrite " + f"a content-addressed object (found {existing})") + except OSError as exc: + raise EmitError(f"object store write failed at {built.object_path}: {exc}") from exc + + built.bundle_path.parent.mkdir(parents=True, exist_ok=True) + # Staged under a name metrics does not rglob, so a failure before the record is + # written cannot leave an orphan green bundle for FCR to count. + staged = built.bundle_path.with_name(built.bundle_path.name + ".staged") + _atomic_write_text(staged, built.bundle_text) + + built.record_path.parent.mkdir(parents=True, exist_ok=True) try: - _atomic_write_text(record_path, json.dumps(record, indent=2, sort_keys=True) + "\n") + _atomic_write_text(built.record_path, built.record_text) except BaseException: staged.unlink(missing_ok=True) raise - os.replace(staged, bundle_path) - return {"bundle": bundle_path, "evidence": record_path, "sha256": record["sha256"]} + os.replace(staged, built.bundle_path) + return {"bundle": built.bundle_path, "evidence": built.record_path, + "sha256": built.sha256, "object": built.object_path} diff --git a/loop/evidence.py b/loop/evidence.py index 17f3957..b56f61d 100644 --- a/loop/evidence.py +++ b/loop/evidence.py @@ -1,11 +1,21 @@ """loop-engineer/evidence@1 — hashed evidence + artifact provenance. ``loop doctor`` discovers and validates records from the declared location -``.loop/evidence/*.json`` (reference/repo-os-contract.md #17) and reports -``self_verified_evidence`` / ``missing_evidence_record``. It does NOT yet -hash-verify the artifacts those records reference, and it never compares a -recorded digest against anything — ``verify_evidence()`` below is the explicit, -caller-invoked check. That wiring is the evidence-wiring slice. +``.loop/evidence/*.json`` (reference/repo-os-contract.md #17), reports +``self_verified_evidence`` / ``missing_evidence_record``, and — since the +evidence-wiring release — composes ``verify_evidence()`` below over every +structurally-valid record, so a referenced artifact that does not hash to the +digest its record declares fails doctor as ``hash_mismatch``. Doctor also +compares the latest record per task against the live TASKS.json goalpost +(``policy_digest_mismatch``), and re-hashes whatever an event bound into the +chain (``evidence_chain_mismatch`` / ``missing_bound_evidence``). + +What is still NOT checked here: ``code_digest`` is never re-hashed against the +verifier file (a verify script legitimately changes between runs, and an +unbaselined comparison would fire on every honest edit), and none of this proves +provenance — a hand-written record whose pointer resolves and whose digests are +self-consistent is indistinguishable from one a dispatch produced. Verification +proves the pointer, never the producer. """ from __future__ import annotations @@ -15,7 +25,7 @@ import os import re import stat -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Mapping from .contract import ContractIssue, _resolve_requested_mode, _schemas_dir @@ -23,10 +33,25 @@ EVIDENCE_SCHEMA_ID = "loop-engineer/evidence@1" +VERIFY_BUNDLE_KIND = "verify-bundle" _URI_PATTERN = re.compile(r"^(?!/)(?![A-Za-z][A-Za-z0-9+.\-]*://).+$") _SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +def verify_bundle_is_green(bundle: Mapping[str, Any]) -> bool: + """The repo's ONE green-marker rule for a verify bundle. + + A bundle is green when it says so explicitly — ``outcome == "PASS"`` or + ``passed is True``. A bundle carrying only a numeric ``score`` reads RED: a + score is not a verdict, and treating one as a pass is exactly the false + completion this kernel exists to refuse (the rule originated in + ``scripts/metrics.py``, which now imports it rather than restating it). + """ + if not isinstance(bundle, Mapping): + return False + return str(bundle.get("outcome", "")).upper() == "PASS" or bundle.get("passed") is True + + class EvidenceError(ValueError): """The workspace-root precondition for evidence verification was not met.""" @@ -217,3 +242,91 @@ def verify_evidence(evidence: Mapping[str, Any], *, workspace_root: str | Path) def artifact_object_path(workspace_root: str | Path, sha256: str) -> Path: """Return evidence@1's content-addressed artifact location without I/O.""" return Path(workspace_root) / ".loop" / "artifacts" / "objects" / sha256[:2] / sha256 + + +#: A chain-bound path is attacker-nameable — any event may declare any string. A gate +#: must therefore never perform an unbounded read on one. 64 MiB is far above any +#: artifact this kernel writes and far below "read whatever is at the other end". +MAX_BOUND_ARTIFACT_BYTES = 64 * 1024 * 1024 + +_DRIVE_PREFIX = re.compile(r"^[A-Za-z]:") + + +def _lexical_escape(rel: object) -> str | None: + """Why this declared path is not workspace-relative — decided with ZERO I/O. + + Runs before anything is opened, so an escaping path is reported rather than read. + """ + if not isinstance(rel, str) or not rel.strip(): + return "is not a non-empty path" + if "\\" in rel: + return "contains a backslash, which is not a workspace-relative POSIX path" + if _DRIVE_PREFIX.match(rel): + return "names a drive letter" + pure = PurePosixPath(rel) + if pure.is_absolute(): + return "is an absolute path" + if ".." in pure.parts: + return "traverses out of the workspace with '..'" + return None + + +def hash_bound_artifact( + workspace_root: str | Path, rel: object, *, max_bytes: int = MAX_BOUND_ARTIFACT_BYTES, +) -> tuple[str, str]: + """Containment-check a chain-bound path, then stream-hash it under a cap. + + Returns ``(code, detail)``: + + * ``("ok", )``; + * ``("escape", )`` — the path is not inside the workspace. For a lexical + escape NOTHING on disk was touched; a symlinked escape is caught after + resolution, exactly as ``verify_evidence`` catches it; + * ``("unreadable", )`` — contained, but its bytes could not be hashed: + absent, not a regular file (so ``/dev/zero`` and friends are refused rather + than read forever), or larger than ``max_bytes``. + """ + escape = _lexical_escape(rel) + if escape is not None: + return "escape", escape + root = Path(workspace_root) + try: + resolved = (root / str(rel)).resolve(strict=True) + except (OSError, ValueError, RuntimeError): + return "unreadable", "is absent or cannot be resolved" + try: + resolved.relative_to(root.resolve()) + except ValueError: + return "escape", "resolves outside the workspace (a symlinked component)" + try: + fd = os.open(resolved, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0)) + except OSError: + return "unreadable", "is absent or cannot be opened" + try: + file_stat = os.fstat(fd) + if not stat.S_ISREG(file_stat.st_mode): + return "unreadable", "is not a regular file" + if file_stat.st_size > max_bytes: + return "unreadable", (f"is {file_stat.st_size} bytes, above the {max_bytes}-byte " + f"bound-artifact read cap, so its digest was not computed") + digest = hashlib.sha256() + read = 0 + duplicate = os.dup(fd) + try: + source = os.fdopen(duplicate, "rb") + except OSError: + os.close(duplicate) # fdopen never took ownership + raise + with source: + while chunk := source.read(64 * 1024): + read += len(chunk) + if read > max_bytes: + return "unreadable", (f"grew past the {max_bytes}-byte bound-artifact " + f"read cap while being hashed") + digest.update(chunk) + except OSError: + return "unreadable", "is absent or cannot be read" + finally: + os.close(fd) + return "ok", digest.hexdigest() diff --git a/loop/integrations.py b/loop/integrations.py index 8b5a736..55ff65c 100644 --- a/loop/integrations.py +++ b/loop/integrations.py @@ -28,7 +28,9 @@ from .completion import ( CompletionPolicyError, criteria_satisfy_completion, + evidence_entry_is_record_shaped, normalize_completion_policy, + policy_requires_verified_evidence, unmet_required_criteria, ) @@ -169,6 +171,12 @@ def body(state: str, reason: str) -> dict: ) if evidence_error is not None: return body("FailedUnverifiable", "invalid evidence artifacts: " + evidence_error) + # Structural half only, fail-closed: a pure projection holds no workspace and must + # not pretend to hash-verify or chain-check what it cites. + if (policy_requires_verified_evidence(normalized_policy) + and not all(evidence_entry_is_record_shaped(item) for item in artifacts)): + return body("FailedUnverifiable", + "green gate but an evidence artifact is not verified evidence — cannot certify") if not artifacts: return body("FailedUnverifiable", "green gate but no evidence artifacts — cannot certify") if not outcome.reached_end: diff --git a/loop/reducer.py b/loop/reducer.py index e88ef7e..4860716 100644 --- a/loop/reducer.py +++ b/loop/reducer.py @@ -5,7 +5,9 @@ from typing import Any, Iterable, Mapping from . import chain, fsm -from .completion import CompletionPolicyError, criteria_satisfy_completion, normalize_completion_policy +from .completion import (CompletionPolicyError, criteria_satisfy_completion, + evidence_entry_is_record_shaped, normalize_completion_policy, + policy_requires_verified_evidence) from .contract import TERMINAL_STATES from .events import _structural_validate_event @@ -47,6 +49,13 @@ def _validate_terminal_payload_semantics(payload: Mapping[str, Any]) -> None: evidence = payload.get("evidence") if not isinstance(evidence, list) or not evidence: raise EventReplayError("Succeeded terminal has empty evidence (G1)") + # The STRUCTURAL half only: this fold is pure and holds no workspace, so it can + # check the shape of a citation and nothing more (repo-os-contract §16). + if (policy_requires_verified_evidence(policy) + and not all(evidence_entry_is_record_shaped(item) for item in evidence)): + raise EventReplayError( + "Succeeded terminal declares verified evidence but an entry is not a " + ".loop/evidence record (G1)") def _validate_superseded_payload_semantics(payload: Mapping[str, Any]) -> None: diff --git a/loop/runner.py b/loop/runner.py index 25557f5..daa32a5 100644 --- a/loop/runner.py +++ b/loop/runner.py @@ -11,11 +11,13 @@ from typing import Any, Callable from . import emit +from .completion import DEFAULT_COMPLETION_MODE, VERIFIED_EVIDENCE_MODE +from .contract import _strict_evidence_failure from .events import (EventRowDecodeError, EventStoreOperationalError, SQLiteEventStore, read_event_rows, validate_event) -from .paths import resolve_loop_paths +from .paths import EVIDENCE_DIR_NAME, resolve_loop_paths from .reducer import ChainBreakError, reduce_events -from .runtime import RuntimeStoreError, _read_store +from .runtime import RuntimeStoreError, _read_store, bound_artifact_digests from .verifier import executed_verifier_identity, injected_verifier_identity @@ -175,6 +177,70 @@ def _reconcile_legacy_iteration(target: str | Path, projection: dict[str, Any]) current_id = iteration_id +def _evidence_records_by_task(paths: Any, projection: dict[str, Any]) -> dict[str, list[str]]: + """Map each task id to the evidence records this run's passing iterations left on disk.""" + records: dict[str, list[str]] = {} + for entry in projection.get("runlog_entries", []): + if entry.get("outcome") != "task_passed": + continue + task_id, iteration_id = entry.get("task_id"), entry.get("iteration_id") + if not isinstance(task_id, str) or not isinstance(iteration_id, int): + continue + record = paths.loop_dir / EVIDENCE_DIR_NAME / f"evidence-iter{iteration_id}.json" + if record.is_file(): + records.setdefault(task_id, []).append(record.relative_to(paths.workspace).as_posix()) + return records + + +def _unmet_strict_bar(paths: Any, cited: list[str]) -> str | None: + """Why the cited records cannot back the strict mode, or ``None`` when they can. + + The SAME predicate ``emit.terminate`` enforces, evaluated against the very records + the payload would cite. Existence is not satisfaction: adopting the strict mode + because a file is present would overstate the bar (decision 7) and would re-open the + committed-then-refused seam this slice closes (decision 2), because the writer runs + the predicate afterwards and raises once the event is already durable. + """ + try: + bound = bound_artifact_digests(paths.workspace) + except RuntimeStoreError as exc: + return (f"the event store could not be read ({exc}), so chain-boundness is " + f"unestablished — and unestablished is not proof") + for entry in cited: + detail = _strict_evidence_failure(entry, paths, bound) + if detail is not None: + return f"{entry} {detail}" + return None + + +def _auto_terminal_payload(paths: Any, tasks: list[dict], projection: dict[str, Any]) -> dict[str, Any]: + """Adopt the verified-evidence mode only when this run can honestly satisfy it. + + Choosing the weaker policy silently would be a self-serving downgrade, so the + all_required branch always says WHY — distinguishing a task that produced no record + at all from a record that is present but fails the bar. + """ + payload: dict[str, Any] = { + "state": "Succeeded", "criteria_met": {task["id"]: True for task in tasks}, + "evidence": ["RUNLOG.md"], "false_completion": False, + "completion_policy": {"mode": DEFAULT_COMPLETION_MODE}, + "iteration_id": projection["iteration_id"], + } + records = _evidence_records_by_task(paths, projection) + missing = sorted(str(task.get("id")) for task in tasks if task.get("id") not in records) + if missing: + payload["reason"] = "tasks with no evidence record: " + ", ".join(missing) + return payload + cited = sorted({record for task in tasks for record in records[task["id"]]}) + unmet = _unmet_strict_bar(paths, cited) + if unmet is not None: + payload["reason"] = "evidence records do not meet the verified-evidence bar: " + unmet + return payload + payload["evidence"] = cited + payload["completion_policy"] = {"mode": VERIFIED_EVIDENCE_MODE} + return payload + + def _reconcile_legacy_terminal(target: str | Path, projection: dict[str, Any]) -> None: """Replay the terminal's already-recorded payload into existing emit APIs.""" terminal = projection.get("terminal") @@ -182,11 +248,13 @@ def _reconcile_legacy_terminal(target: str | Path, projection: dict[str, Any]) - return paths = resolve_loop_paths(target) if not paths.terminal.exists(): + reason = terminal.get("reason") emit.terminate( target, state=terminal["state"], criteria_met=terminal["criteria_met"], evidence=terminal["evidence"], + reason=reason if isinstance(reason, str) else "", false_completion=terminal["false_completion"], iteration_id=terminal.get("iteration_id"), completion_policy=terminal.get("completion_policy"), @@ -220,11 +288,7 @@ def dispatch_once( done = done_task_ids(tasks, projection) if all(task.get("id") in done for task in tasks): iteration_id = projection["iteration_id"] - payload = { - "state": "Succeeded", "criteria_met": {task["id"]: True for task in tasks}, - "evidence": ["RUNLOG.md"], "false_completion": False, - "completion_policy": {"mode": "all_required"}, "iteration_id": iteration_id, - } + payload = _auto_terminal_payload(paths, tasks, projection) store = SQLiteEventStore(paths.loop_dir / "events.db") _store_append(store, run_id, "terminal_written", payload, actor="loop.run", expected_sequence=projection["last_sequence"] + 1) @@ -252,17 +316,30 @@ def dispatch_once( "task_id": task["id"], "summary": outcome.summary, } + # Rendered BEFORE the append and written AFTER it. Building first is what lets + # the event carry the digests of the very bytes that will land; the builder + # writes nothing, so a SIGKILL at the pre-commit COMMIT still leaves the tree + # byte-identical (test_crash_injection_before_iteration_event_commit_...). + try: + built = emit.build_verify_evidence( + target, run_id=run_id, iteration_id=iteration_id, task=task, + passed=outcome.passed, summary=outcome.summary, code_identity=code_identity, + executor=executor, verifier_identity=verifier_identity, attempt=attempt, + ) + except emit.EmitError as exc: + raise RunnerError( + f"cannot build the verify evidence for iteration {iteration_id}: {exc}" + ) from exc store = SQLiteEventStore(paths.loop_dir / "events.db") _store_append(store, run_id, "iteration_appended", payload, actor="loop.run", + artifact_hashes=list(built.artifact_hashes), expected_sequence=projection["last_sequence"] + 1) - # Evidence is written AFTER the durable append: writing it first would leave a - # bundle behind a SIGKILL at the pre-commit COMMIT and break the zero-write - # crash pin (test_crash_injection_before_iteration_event_commit_...). try: written = emit.write_verify_evidence( target, run_id=run_id, iteration_id=iteration_id, task=task, passed=outcome.passed, summary=outcome.summary, code_identity=code_identity, executor=executor, verifier_identity=verifier_identity, attempt=attempt, + built=built, ) except (OSError, emit.EmitError) as exc: raise RunnerError( @@ -273,4 +350,4 @@ def dispatch_once( task_id=payload["task_id"], notes=payload["summary"]) return {"ok": True, "action": "dispatched", "task_id": task["id"], "outcome": payload["outcome"], "iteration_id": iteration_id, "run_id": run_id, - "evidence": str(written["evidence"])} + "evidence": str(written["evidence"]), "object": str(written["object"])} diff --git a/loop/runtime.py b/loop/runtime.py index c41bf78..7c994e7 100644 --- a/loop/runtime.py +++ b/loop/runtime.py @@ -302,6 +302,10 @@ def event_consistency_issues( status = status_report(target, mode=mode) replay = replay_report(target, mode=mode) declares_chain_without_columns = _store_declares_chain_without_columns(path) + # Inside the guard with its two siblings: this is a THIRD independent read, and a + # store that becomes unreadable between reads must surface as a typed finding + # rather than an untyped traceback out of doctor_report (R007). + bound_issues = _bound_evidence_issues(target, mode) except RuntimeStoreError as exc: unreadable_issues = [ContractIssue(exc.code, str(exc))] if expect_chain_head is not None: @@ -309,6 +313,7 @@ def event_consistency_issues( "an anchored chain head was supplied but the event store cannot be read")) return {"present": True, "readable": False, "error_code": exc.code}, unreadable_issues issues = list(status["divergence"]) + list(replay["findings"]) + issues.extend(bound_issues) if declares_chain_without_columns: issues.append(ContractIssue( "chain_columns_missing", @@ -330,3 +335,80 @@ def event_consistency_issues( "legal_sequence": replay["legal_sequence"], "chain": {"head": status["chain_head"], "unchained_prefix": status["unchained_prefix"]}, }, issues + + +def _bound_evidence_issues(target: str | Path, mode: str | None) -> list[dict[str, Any]]: + """Re-hash every artifact an event bound into the chain. + + Driven by what each event DECLARES: a legacy event carries an empty + artifact_hashes list and is silent by construction, because the append-only + triggers make a retroactive binding impossible (repo-os-contract.md #22). + + A declared path is attacker-nameable — event@1 constrains it only to a non-empty + string, and the chain covers a binding rather than vouching for it. Every path is + therefore containment-checked BEFORE anything is opened and hashed under a cap + (``loop.evidence.hash_bound_artifact``); an escaping path is reported as + ``bound_evidence_escape``, never read. + """ + from .evidence import hash_bound_artifact + + _, _run_id, events, _validation = _events(target, mode) + workspace = resolve_loop_paths(target).workspace + issues: list[dict[str, Any]] = [] + for event in events: + for entry in event.get("artifact_hashes") or []: + code, detail = hash_bound_artifact(workspace, entry["path"]) + if code == "escape": + issues.append(ContractIssue( + "bound_evidence_escape", + f"event {event['event_id']} (sequence {event['sequence']}) bound " + f"{entry['path']!r}, which {detail} — a bound path that does not " + f"resolve inside the workspace is a finding, not something to read")) + continue + if code == "unreadable": + issues.append(ContractIssue( + "missing_bound_evidence", + f"event {event['event_id']} (sequence {event['sequence']}) bound " + f"{entry['path']} into the chain but it {detail}")) + continue + actual = detail + if actual != entry["sha256"]: + issues.append(ContractIssue( + "evidence_chain_mismatch", + f"{entry['path']} does not match the digest bound at sequence " + f"{event['sequence']}: expected {entry['sha256']}, found {actual} — " + f"the original bytes may remain at " + f".loop/artifacts/objects/{entry['sha256'][:2]}/{entry['sha256']}")) + return issues + + +def bound_artifact_digests(target: str | Path, + mode: str | None = None) -> dict[str, tuple[str, ...]] | None: + """{workspace-relative POSIX path: every DISTINCT sha256 an event bound it at}. + + Conflict-aware by construction. A dict-comprehension keyed on path would collapse + repeat bindings LAST-WINS, which is not a summary but a laundering channel: an + append-only forge that re-binds a tampered path at its new digest would look bound + to this write-time view while ``_bound_evidence_issues`` — which checks PER EVENT — + still reports ``evidence_chain_mismatch`` on the same tree. Returning the full + conflict set keeps the two views in agreement on every tree; the strict bar refuses + a path carrying more than one digest, because ambiguous is not proof. + + Digests are in first-bound order and de-duplicated, so the ordinary case (the same + path bound repeatedly at the same bytes) stays a one-element tuple. + + None means there is no event store, and the caller MUST degrade explicitly and say + so (decision 14) rather than treat absence as satisfaction. An empty dict means a + store exists and bound nothing. An unreadable store raises RuntimeStoreError — an + errored check fails, it never skips (R007). + """ + if not (resolve_loop_paths(target).loop_dir / "events.db").is_file(): + return None + _, _run_id, events, _validation = _events(target, mode) + digests: dict[str, list[str]] = {} + for event in events: + for entry in event.get("artifact_hashes") or []: + seen = digests.setdefault(entry["path"], []) + if entry["sha256"] not in seen: + seen.append(entry["sha256"]) + return {path: tuple(seen) for path, seen in digests.items()} diff --git a/loop/simulate.py b/loop/simulate.py index d5eac40..27827d6 100644 --- a/loop/simulate.py +++ b/loop/simulate.py @@ -67,13 +67,10 @@ def _predict(paths: Any, projection: dict[str, Any]) -> dict[str, Any]: if task is None: done = runner.done_task_ids(tasks, projection) if all(candidate.get("id") in done for candidate in tasks): - payload = { - "state": "Succeeded", - "criteria_met": {candidate["id"]: True for candidate in tasks}, - "evidence": ["RUNLOG.md"], "false_completion": False, - "completion_policy": {"mode": "all_required"}, - "iteration_id": projection["iteration_id"], - } + # Built by the runner's own function, never restated here: a second copy of + # the auto-terminal payload silently breaks the predicted-equals-real pin + # (test_simulate_predicted_terminal_payload_matches_real_..._exactly). + payload = runner._auto_terminal_payload(paths, tasks, projection) return {**empty, "action": "would_write_terminal", "predicted_terminal": payload} return {**empty, "action": "would_block"} diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index a905d0c..b818e83 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -622,9 +622,15 @@ exception. `VerifierExecutionError`, `VerifierNotImplementedError`, and attempted. **Event types:** `contract_opened | iteration_appended | receipt_appended | -terminal_written` — one-to-one with `loop.emit`'s four writer operations +terminal_written | terminal_superseded | approval_requested | +approval_resolved | run_paused | run_resumed` — nine members, matching +`loop.events.EVENT_TYPES` and the `type` enum of `schemas/event.schema.json`. +The **first four** correspond to `loop.emit`'s writer operations (`open_contract`/`append_iteration`/`append_receipt`/`terminate`), so a -future write-through migration targets an already-matching payload shape. +future write-through migration targets an already-matching payload shape. The +remaining five have no `emit` writer by design: `terminal_superseded` is the +administrative correction of §18, and the four run-control types are §19's +projection primitives. **Two-layer enforcement, deliberately split:** the store validates event@1 envelope/payload *shape* only (`loop/events.py::validate_event`, both @@ -751,6 +757,19 @@ appended after a chained prefix is reported as `event_chain_broken` and is unrepairable, because UPDATE is trigger-blocked. Pin your loop-engineer (and action) version per store. +**Bound artifacts (v0.11.0+).** `artifact_hashes` is field eleven of the +twelve-field preimage, so any `{path, sha256}` an appender places there is +already covered by `event_hash` and therefore by `--expect-chain-head`. Binding +evidence needs **no new event type and no second append**: since the +evidence-wiring release a verified dispatch binds three entries on the one +`iteration_appended` event it already writes — the verify bundle, its evidence@1 +record, and the bundle's content-addressed object (§17). The digests are +computed **before** the append and the files are written **after** it. An event +that binds nothing — every event written before that release, and every append +by a foreign writer — is silent by construction, and stays silent: the +append-only triggers forbid the UPDATE a retroactive binding would need, exactly +as `loop migrate` refuses to backfill chain hashes (§22). + ### Integrity boundary The chain is **tamper-evident relative to an anchored head**. That is a @@ -785,7 +804,13 @@ trailing events leaves a shorter but internally valid chain. `deterministic` and `legal_sequence` stay `true`. A future standalone event-store cross-check — a new issue code appended directly to `issues`, as `chain_columns_missing` is — would not move any of those, so it must be added - to this pin's assertions when it is introduced.) + to this pin's assertions when it is introduced. The evidence-wiring release + introduced exactly two such codes, `evidence_chain_mismatch` and + `missing_bound_evidence`; this pin's assertions were **not** widened to name + them. The stronger statement lives in the binding suite instead: + `scripts/test_adversarial_evidence_binding.py::test_a_full_rewrite_of_artifacts_and_store_is_not_caught_without_an_anchor_pinned` + rewrites the artifacts, the object, the bound digests and the chain, and + asserts the whole report is `ok` with **zero** issues.) - **A chain-column downgrade.** Dropping `event_hash`/`prev_event_hash`, or rebuilding the store without them, silently downgrades a chained history to an unchained one. An unchained or legacy doctor report is *not* proof of @@ -835,10 +860,15 @@ workspace, rejects traversal and symlink escapes, and verifies the file hash. `.loop/artifacts/objects/` without writing it. **Scope boundary:** `loop doctor` reads evidence@1 records from the declared -location `.loop/evidence/*.json` and validates them; it does **not** yet -hash-verify the artifacts they reference, does **not** compare any recorded -digest against anything, and `Succeeded` still requires non-empty evidence -*paths*, not verified hashes. Those tightenings are the next slice. +location `.loop/evidence/*.json`, validates them, **hash-verifies the artifact +each one references**, and compares the `policy_digest` of the **latest record +per task** against the live `TASKS.json` entry. Under the default +`all_required` completion policy `Succeeded` still requires non-empty evidence +*paths*; under the opt-in `all_required_verified_evidence` policy it requires +every entry to be a hash-verified evidence@1 record that — wherever an event +store exists to bind against — some event bound into the hash chain, and whose +recorded goalpost is the live one. `code_digest` is still never re-hashed +against the verifier file, and no check here proves who produced a record. **Artifact provenance:** `kind` remains an open vocabulary (for example, `verify-bundle`, `log`, `diff`, `screenshot`, or `report`), while `produced_by` @@ -889,8 +919,10 @@ make the digest noise; `id` binds *which* goalpost the digest names, and `depends_on` binds its declared ordering, so both are included. The digest binds the criterion **reference**, not the criterion **text** — -editing `SPEC.md`'s acceptance wording leaves it unchanged. Binding criterion -text is the evidence-wiring slice. +editing `SPEC.md`'s acceptance wording leaves it unchanged. The evidence-wiring +release did **not** change that: criterion text remains bound by nothing, pinned +by `test_criterion_text_is_still_unbound_pinned`. Binding evidence bytes does +not bind intent. **Conformance vector.** Over the task entry @@ -923,14 +955,25 @@ to the bundle bytes via `sha256`). An evidence record MUST NOT be named bundle with no green marker, i.e. a phantom failing gate. `verifier.source` is `declared_command` only when the task's declared `verify` command was executed. A bundle whose source is `injected_callable` carries a -caller-supplied verdict and is not gate evidence. `scripts/metrics.py` does not -read `verifier.source` today, so an injected-callable bundle counts toward FCR -exactly like a declared-command bundle — enforcing that distinction is Slice 3 -scope. Before this slice the runner wrote no bundles at all, so a -runner-driven contract's FCR input set changes with this release. `loop -simulate` predicts decisions, not writes: it reports `legacy_sync_would_write` -because that write is conditional, but it does not enumerate the bundle and -record a dispatch always writes. +caller-supplied verdict and is not gate evidence. `scripts/metrics.py` enforces +that distinction: a bundle whose `verifier.source` is **explicitly** +`injected_callable` is excluded from the FCR input set before any metric is +derived, and the excluded filenames are listed under +`provenance.injected_verifier_bundles` so the exclusion is visible rather than a +silent FCR shift. A bundle carrying no `verifier` block at all has an *unknown* +source and still counts — grandfathering by absence, not by guess. `source` is a +string the writer declares, not a fact metrics can check, so hand-writing +`declared_command` restores gate-evidence status; that limitation is pinned by +`test_metrics_still_counts_a_hand_written_declared_command_bundle_pinned`. +Two input-set changes land together in this release and are stated rather than +absorbed: before it the runner wrote no bundles at all, and injected-callable +bundles counted — so a runner-driven contract's FCR input set moves on upgrade. +`loop simulate` predicts decisions, not writes: it reports +`legacy_sync_would_write` because that write is conditional, but it does not +enumerate the bundle, record and content-addressed object a dispatch always +writes, and it does not enumerate the object store. A boolean that is `True` on +every dispatch carries no information, so `_empty_prediction` gains no field +here (§20). **The partition.** `visible` defaults to the task's `criterion_ref`; `holdout` is empty unless the task declares `holdout_criteria`; both fields @@ -948,40 +991,279 @@ fails. On the `loop run` path both identities are operator-supplied (`--executor`, `--verifier-identity`); their defaults (`unattributed`, `loop.run`) never collide, so a default run cannot manufacture the finding. -**The integrity boundary, in four honest tiers** — not a single "surfaces / -does not surface" pair: +### Bound evidence (v0.11.0+) + +**The bound set, in the order the writer emits it.** A verified dispatch binds +exactly three `{path, sha256}` entries into the `artifact_hashes` of the one +`iteration_appended` event it already writes (§16), in this order: + +1. `.loop/artifacts/verify-iter.json` — the verify bundle, at the bundle's + own sha256; +2. `.loop/evidence/evidence-iter.json` — the evidence@1 record, at the + record file's sha256; +3. `.loop/artifacts/objects//` — the content-addressed **object**, a + byte-identical copy of the bundle, at the bundle's sha256 again. + +**The object's location is derived, never declared.** It is +`artifact_object_path(workspace, record["sha256"])` — +`.loop/artifacts/objects//` — a pure function of a +digest evidence@1 already carries, so **evidence@1 gains no field** and a +third-party reader locates the object from `record["sha256"]` alone. The object +is the recovery source when the friendly-named bundle is swapped: the swap fails +doctor as `evidence_chain_mismatch` *and* the original bytes are still on disk. +Deleting the object first does not clear the path — the object is itself a bound +artifact, so its removal is reported as `missing_bound_evidence`. + +**Objects are created once and never overwritten.** The write uses the same +hard-link create-once primitive as the immutable terminal record. An existing +object with identical bytes is idempotent success (a re-run of the same +dispatch); an existing object with **different** bytes for the same digest is a +typed `EmitError` naming the digest, because a corrupted or colliding object +store must never be silently accepted. + +**Ordering: build (no I/O) → append (binds) → write (object, staged bundle, +record, replace).** The builder is pure — it renders the exact bytes and their +digests and touches nothing — so the event commits to digests that describe what +is about to land. Two consequences, both deliberate: a **build failure commits +nothing** (a non-canonicalizable task or an invalid record is refused before any +file exists), and a **crash after the durable append** leaves an event naming +artifacts that are absent — reported as `missing_bound_evidence` rather than +passing as a silent gap. The crash window still exists; what changed is that it +is no longer invisible. + +### The verified-evidence completion mode (v0.11.0+) + +`completion_policy.mode` accepts a second value, +`all_required_verified_evidence`, alongside the default `all_required`. The +criteria half is unchanged — every declared criterion must still be `True`; the +mode raises the **evidence** bar only. It is **opt-in and never retroactive**: +`normalize_completion_policy(None)` still returns `{"mode": "all_required"}`, so +every record written before this release — including both shipped examples, +which declare no policy at all — keeps the old bar forever +(`test_the_verified_evidence_mode_is_opt_in_and_not_retroactive_pinned`). + +**Four enforcement layers, each as strong as it can honestly be.** + +| Layer | Reaches | Enforces | +|---|---|---| +| `emit.terminate` (write time) | workspace + event store | the full bar; failure is a typed `EmitError` | +| `loop doctor` / `loop.contract` (read time) | workspace + event store | the same bar, reported as `unverified_evidence_terminal` | +| `loop.reducer` (pure fold) | nothing — no I/O | shape only: every entry is a workspace-relative `.loop/evidence/*.json` path | +| `loop.integrations.to_terminal_state` (pure projection) | nothing — no workspace | the same shape half, returning `FailedUnverifiable` rather than raising | + +The two workspace-bearing layers share **one predicate** +(`loop.contract._strict_evidence_failure`); `emit.terminate` imports it rather +than restating it, because two hand-written copies of a four-part check drift +and a drift here is a silent false completion. That predicate is a strict +**superset** of the pure half: it applies the reducer's own +`evidence_entry_is_record_shaped` to the cited entry first, so the writer can +never accept a terminal that its own replay would refuse. The two pure layers +check shape and nothing else because they hold no filesystem — making them +*appear* to enforce binding would be the pretending this contract refuses. + +**What the mode proves, exactly.** For every cited entry: (1) the entry resolves +to a readable evidence@1 record that validates, and the artifact its `uri` names +resolves inside the workspace and hashes to the digest the record declares; +(2) that artifact is a **verify bundle whose verdict is a pass**; (3) **when an +event store exists**, those record bytes are the bytes some event bound into the +chain, at that digest, and at exactly **one** digest; and (4) when the record +names a task still in `TASKS.json`, its recorded `policy_digest` equals the live +goalpost. A goalpost that cannot even be **computed** is a failure, not a skip — +unestablished is not agreement. An event store that cannot be **read** is likewise +a refusal, never a pass. The message names which of the four sub-checks failed. + +**Check (2), the verdict, in full.** Authenticity is not success. A dispatch whose +verifier FAILS still writes and binds a perfectly genuine record — of a failure — +so the bar asks what the artifact SAYS as well as whether it is real. Green is the +repo's one green-marker rule, `loop.evidence.verify_bundle_is_green`: `outcome == +"PASS"` **or** `passed is true`. A bundle carrying only a numeric `score` reads +**red**, here and in `scripts/metrics.py`, which imports the same predicate — a +bundle that reads red to the FCR gate can never read green to the completion gate. +`kind` is an open vocabulary, and a record that is **not** a `verify-bundle` (a +log, a diff, a screenshot) carries no verdict this layer can read: it is refused, +because an artifact with no verdict cannot show that anything passed. An +unreadable or unparseable verdict is a refusal, never a skip. + +**Check (3), and why ambiguity is refused.** The write-time view +(`loop.runtime.bound_artifact_digests`) reports every **distinct** digest a path +was bound at, not the last one. Collapsing repeat bindings would be a laundering +channel rather than a summary: an append-only forge that re-binds a tampered path +at its new digest would look bound to the writer while the per-event walk still +reported `evidence_chain_mismatch` on the same tree. A path bound at two or more +different digests is therefore refused outright. + +The scope of that refusal is the **cited** evidence, and no wider. The write-time +bar examines each entry a `Succeeded` terminal cites and the artifact that entry's +`uri` names; it does not walk the other paths those events happened to bind. So a +tree can carry an ambiguously-bound artifact that no terminal cites, and the +writer will not see it while `loop doctor`'s per-event walk still reports +`evidence_chain_mismatch`. The two layers agree about every path a terminal +rests on, which is the claim; they are not two spellings of the same sweep. + +**What it does not prove, stated without hedging.** It does not prove that a +human, a runner, or anything at all *produced* the record: a hand-written record +whose pointer resolves and whose digests are self-consistent satisfies the mode +in a **store-less** contract, because there is nothing there to bind against +(`test_the_strict_mode_accepts_a_hand_written_record_in_a_store_less_contract_pinned`). +It does not prove the chain itself was not rewritten: a worker who can rewrite +`.loop/` can rewrite the store, so **without `--expect-chain-head` a full rewrite +of artifacts *and* chain satisfies the strict mode** +(`test_a_full_rewrite_satisfies_the_strict_mode_without_an_anchor_pinned`). The +mode's strength is therefore **store-dependent and anchor-dependent**. + +**Deleting the store is not louder — it is quieter.** Measured, not assumed: a +sidecar-free read (v0.10.0) leaves no `-wal`/`-shm` behind, and doctor's +`missing_event_store` tripwire is gated on exactly that residue, so removing +`.loop/events.db` after an ordinary run leaves plain `loop doctor` at `ok: true` +with **zero issues**, and `emit.terminate` then accepts a hand-written record +under the strict mode. `missing_event_store` catches only the sloppier deletion +that leaves a sidecar. The control that does hold is the external anchor: +`loop doctor --expect-chain-head ` reports `chain_anchor_mismatch` against +the deleted store and stays quiet against an intact one. Without an external +anchor, deleting the store silently disables this mode's chain-boundness check. + +**A deliberate asymmetry with doctor's general goalpost check.** Doctor compares +the **latest record per task** (see the tier list below); the terminal check +compares **every cited record**. "Latest" orders numbered +`evidence-iter.json` records by iteration and ranks every **unnumbered** record +below all of them, ties broken by filename — unnumbered records are ranked in, not +dropped, so a task whose only record carries no iteration id is still compared. Citing a record in a `Succeeded` terminal is a +present-tense claim that *this* record backs completion now, so a stale goalpost +in a cited record is a false completion, not an artifact of history. + +**`loop run`'s auto-terminal adopts the mode only when it can satisfy it**, and +never on file existence: it evaluates the same shared predicate against the very +records it would cite, **before** appending the terminal event. When it cannot, +it writes `{"mode": "all_required"}` with `evidence: ["RUNLOG.md"]` and says why +in `reason` — either `tasks with no evidence record: …` or `evidence records do +not meet the verified-evidence bar: `. Downgrading silently +would be a self-serving choice; both branches are pinned. + +### Upgrade notes — four behavioural changes, stated without hedging + +These are not opt-in, and two of them can turn a contract that was clean on the +previous release red on this one. They are recorded here rather than left to be +discovered. + +**1. `policy_digest_mismatch` is NOT opt-in, and its remedy is currently +unsatisfying.** The verified-evidence *completion mode* is opt-in; this doctor +check is not. A contract that was doctor-clean at v0.10.0 goes **red** on upgrade +as soon as anyone makes an ordinary goalpost edit — changing a task's `verify`, +`criterion_ref`, `depends_on` or `id` after a verification was recorded. The +documented remedy is "re-verify to record the current goalpost", and in a +loop whose done-ness comes from the event log the runner will not re-verify a task +it already considers done, so re-verification does not generally help. Measured, +the reliable route back to green is to **rename the task** — which is precisely the +evasion the tier list below admits this check cannot catch (a record naming a task +absent from `TASKS.json` is never compared). So the check is honest about a moved +goalpost and weak against a determined one, and the operator's real options today +are: edit goalposts *before* the verification that records them, re-run the +dispatch so a fresh record is written, or accept the finding as the true statement +that the declared goalpost moved after the work was verified. A first-class +"re-baseline this task's goalpost" affordance does not exist yet. + +**2. The `completion_policy` enum widening is forward-INCOMPATIBLE as a hard +error.** `all_required_verified_evidence` is a new enum member, so an **older** +kernel reading a terminal that declares it reports `invalid_completion_policy` +and `schema_violation` — it does not degrade to the old bar, it fails. This +matters concretely for the README's pinned `@v0.10.0` action and for +`uvx loop-engineer@0.10.0`: a gate pinned to the older release will reject a +contract written by the newer one. Pin the gate and the writer to the same +release, or keep writing `all_required` until the gate is upgraded. + +**3. `os.link` now runs once PER DISPATCH, not once per run.** The +content-addressed object is placed with the same create-once hard link as the +immutable terminal record, and every dispatch places one. On a filesystem without +hard-link support (some network and container-overlay mounts, and any FAT-family +volume) the link fails, so **every dispatch becomes a committed-then-failed +iteration**: the durable event is already appended when the object write raises. +Previously the same limitation existed but was hit once, at terminal write. There +is no fallback copy path by design — a non-atomic create would reopen the +overwrite race the hard link closes — so this is a known limitation of running a +loop on such a filesystem, not a configuration option. + +**4. A bound artifact above the read cap fails doctor, and there is no knob.** The +binding walk refuses to hash a bound path larger than `MAX_BOUND_ARTIFACT_BYTES` +(64 MiB) and reports `missing_bound_evidence`, because a gate must not perform an +unbounded read on a path an event names. A loop that legitimately binds an +artifact above that size — a large log, a coverage dump — therefore goes red with +no in-product remedy and no configuration option, the same un-satisfiable shape as +note 1. Bind a digest of the large artifact rather than the artifact itself, or +keep it out of `artifact_hashes`. + +### The integrity boundary, in four honest tiers + +Not a single "surfaces / does not surface" pair: - **Fails `loop doctor`:** a record declaring self-verification (`self_verified_evidence`); a runner-written bundle whose record is absent (`missing_evidence_record`); a malformed or unparseable record - (`invalid_evidence` — an errored check fails, it never skips). + (`invalid_evidence` — an errored check fails, it never skips). Added by the + evidence-wiring release: a discovered record whose referenced artifact does + not match its `sha256` (`hash_mismatch`) or does not resolve at all + (`missing_evidence_path`, surfaced verbatim out of `verify_evidence`); the + **latest** record for a task whose recorded goalpost disagrees with — or + cannot be compared against — the live `TASKS.json` entry + (`policy_digest_mismatch`); a chain-bound artifact whose bytes changed + (`evidence_chain_mismatch`) or vanished (`missing_bound_evidence`); and a + `Succeeded` terminal under `all_required_verified_evidence` whose cited + evidence fails the bar (`unverified_evidence_terminal`). - **Recorded for later comparison, not checked by any shipped surface:** - `policy_digest` and `code_digest`. Nothing in v0.11.0 compares either - against the live TASKS.json entry or re-hashes the verifier file, so a moved - goalpost or a swapped verify script is *legible in a diff of two records* - and nothing more. Automated comparison is the evidence-wiring slice. -- **Detectable only by explicitly calling `verify_evidence()`:** a swapped - bundle. `loop doctor` does not hash-verify referenced artifacts in this - release. + `code_digest`, and now only `code_digest`. Nothing re-hashes the verifier + file, and nothing will without a declared baseline: a verify script + legitimately changes between runs, so an unbaselined comparison would fire on + every honest edit. A swapped verify script therefore remains *legible in a + diff of two records* and nothing more + (`test_doctor_does_not_re_hash_the_verifier_file_pinned`). **`policy_digest` + has left this tier** — the evidence-wiring release compares it against the + live goalpost. +- **Detectable only by explicitly calling `verify_evidence()`:** **this tier is + now empty.** The evidence-wiring release composes that call inside + `loop doctor` for every discovered record, so a swapped bundle — the one + member this tier ever had — now fails doctor as `hash_mismatch` in the first + tier above. - **Not surfaced at all:** a worker that writes a false `verified_by.by`; a - worker that omits its executor identity; a hand-written record whose - digests are invented (doctor validates their *shape*, never their truth); a - rewrite or deletion of the record **and** its bundle together. **Both - artifacts are plain files outside the hash chain: a worker with write - access to `.loop/` can rewrite or remove them and `loop doctor` will not - notice.** Binding the record digest into the chain requires a new event - type and is the evidence-wiring slice. A record whose referenced bundle is - absent also passes `loop doctor` clean: `missing_evidence_record` is - one-directional (it walks bundles looking for their record, never the - reverse), so doctor never checks that a record's `uri` resolves. And an - iteration committed to the event log with no evidence written at all — the - sanctioned crash window between the durable append and the evidence write — - is undetectable, because the tripwire needs a bundle to fire before it can - notice anything missing. Closing both gaps is Slice 3 scope. + worker that omits its executor identity; a hand-written record whose pointer + resolves and whose digests are self-consistent (verification proves the + *pointer*, never the *provenance* — + `test_a_hand_written_record_pointing_at_a_real_file_still_passes_pinned`); the + *text* of a criterion, which `policy_digest` never covered and still does not. + Plus the residuals this release's own binding leaves, each pinned by name: + - a record written through the **writer API** (`emit.write_verify_evidence`) + appends no event, so nothing binds it and a later rewrite is invisible + (`test_a_record_written_outside_a_dispatch_is_never_chain_bound_pinned`); + - a **legacy** event carries `artifact_hashes: []` and can never be bound + retroactively — the append-only triggers forbid the UPDATE a backfill would + need — so a pre-release iteration whose evidence was deleted stays + undetectable at this layer forever + (`test_a_legacy_iteration_event_can_never_be_bound_retroactively_pinned`); + - an **older** record for a task whose goalpost has since moved is not + compared, because comparing every record would make each honest + re-verification a permanent failure + (`test_an_older_record_for_a_moved_goalpost_is_not_compared_pinned`); + - a hand-written record **satisfies `all_required_verified_evidence` in a + contract that has no event store**, because there is nothing to bind + against — and deleting a store from a run that had one is *quieter*, not + louder: sidecar-free reads leave no residue for the `missing_event_store` + tripwire, so plain doctor reports `ok` with zero issues. The external + anchor is the compensating control + (`test_the_strict_mode_accepts_a_hand_written_record_in_a_store_less_contract_pinned`); + - deleting the artifacts, the records, the store **and** its sidecars leaves a + contract that never ran, and doctor is clean and correct to be — detection + needs something outside the tree + (`test_deleting_every_artifact_and_the_store_leaves_a_clean_doctor_pinned`, + whose own positive control is the anchor); + - and — the headline — **a worker who can rewrite `.loop/` can rewrite the + store, so without `--expect-chain-head` a full rewrite of artifacts *and* + chain verifies clean, including under the strict completion mode** + (`test_a_full_rewrite_of_artifacts_and_store_is_not_caught_without_an_anchor_pinned`, + `test_a_full_rewrite_satisfies_the_strict_mode_without_an_anchor_pinned`). This does not prove independence. It surfaces **declared** self-verification, -and it records — honestly, with nulls where the process could not know — -what verified the work. +and it records — honestly, with nulls where the process could not know — what +verified the work. This does not make evidence tamper-proof. It binds evidence +bytes into a chain someone outside the worker's trust domain can anchor, and it +fails closed when what is on disk is not what was verified. --- @@ -995,12 +1277,11 @@ non-empty `justification`, and `{by, at}` `authority`, and its `causation_id` must identify the terminal event it corrects; chained corrections therefore remain auditable without replacing any record. -**Scope boundary:** this is a fifth event type with no corresponding -`loop.emit` writer operation — deliberately: unlike the other four, -`terminal_superseded` is administrative and event-log-only; §16's -“one-to-one with `loop.emit`'s four writer operations” describes the other four -types and predates this addition. It is not file-based `terminal@1` replacement -or an `emit`/`doctor` workflow. +**Scope boundary:** this event type has no corresponding `loop.emit` writer +operation — deliberately: unlike the first four types, `terminal_superseded` is +administrative and event-log-only (§16's type list records which four types map +to writer operations and which five do not). It is not file-based `terminal@1` +replacement or an `emit`/`doctor` workflow. **Domain enforcement:** the EventStore validates envelope and payload shape only; the reducer alone admits this type after a terminal, verifies its @@ -1105,9 +1386,12 @@ only finding is the anchor. A present, readable store adds `"event_store": {"present": true, "readable": true, "run_id", "event_count", "state_json_agrees", "deterministic", "legal_sequence", "chain"}`; any of `state_field_mismatch`, `desynced_terminal_window`, `terminal_state_mismatch`, -`illegal_event_sequence`, `event_chain_broken`, `chain_columns_missing`, or -`chain_anchor_mismatch` fails doctor (`ok: false`) with the identical issue -code the `status`/`replay` verbs already use. A store that cannot be read at +`illegal_event_sequence`, `event_chain_broken`, `chain_columns_missing`, +`evidence_chain_mismatch`, `missing_bound_evidence`, `bound_evidence_escape`, or +`chain_anchor_mismatch` fails doctor (`ok: false`). The findings that come from +the composed verbs keep the identical issue code those verbs already use; +`chain_columns_missing`, `evidence_chain_mismatch`, `missing_bound_evidence` and +`bound_evidence_escape` are doctor's own store-gated checks, appended to the same list. A store that cannot be read at all — `corrupt_store`, `empty_store`, `invalid_event`, or `ambiguous_run_id` — also fails doctor rather than being silently skipped; `"event_store"` reports `{"present": true, "readable": false, "error_code": }` in that case. @@ -1139,13 +1423,36 @@ it is the honest statement of how much of the log the chain does not cover. | `missing_event_store` | `events.db` is absent but `-wal`/`-shm` sidecars remain — the store was deleted. Distinct from the pre-existing `missing_store`, which `status`/`replay`/`run`/`migrate` raise when a verb that *requires* a store is pointed at a workspace that has none; `missing_event_store` is a doctor finding about a store that evidently once existed. | | `self_verified_evidence` | A discovered evidence@1 record declares `produced_by.executor == verified_by.by` (strip+casefold) — the producer verified its own work. Enforces the independence rule of `reference/safety-and-approvals.md` §5, which was prose-only before v0.11.0. | | `missing_evidence_record` | A runner-written verify bundle `.loop/artifacts/verify-iter.json` exists with no matching `.loop/evidence/evidence-iter.json`. Residue of a removed provenance record, in the same family as `missing_event_store`. Fires only when a bundle is present, so an absent-everything contract stays byte-identical. | +| `evidence_chain_mismatch` | An artifact whose digest an event bound into the hash chain no longer matches its bytes on disk. The message names the digest the chain committed and the digest found; the original bytes may still remain in the content-addressed object store at `.loop/artifacts/objects//`. | +| `missing_bound_evidence` | An event bound an artifact path into the chain and that path is now absent or unreadable — a deleted bundle/record pair, a deleted object, the sanctioned crash window between the durable append and the evidence write, or a path that resolves to something the gate refuses to read (a non-regular file such as a device or FIFO, or a file above the 64 MiB bound-artifact read cap). Fires only for events that bound something: a legacy event binds nothing and is silent by construction. | +| `bound_evidence_escape` | An event bound a path that is not inside the workspace — absolute, drive-lettered, backslashed, `..`-traversing, or resolving outside through a symlinked component. `event@1` constrains `path` only to a non-empty string and the chain covers a binding rather than vouching for it, so the binding walk containment-checks every declared path **before** opening anything: an escaping path is reported, never read. | +| `policy_digest_mismatch` | The **latest** evidence record for a task records a `policy_digest` that is not the live `TASKS.json` goalpost — the declared `verify`/`criterion_ref`/`depends_on`/`id` changed after the verification. Also fires when the live entry cannot be canonicalized at all, because a comparison that cannot run has not passed. Re-verify to record the current goalpost. | +| `unverified_evidence_terminal` | A `Succeeded` terminal declaring `completion_policy.mode: all_required_verified_evidence` has an evidence entry that fails one of the mode's four checks: it is not a hash-verified evidence@1 record; or it does not point at a `verify-bundle` whose verdict is a pass; or (when an event store exists) no event bound those bytes into the chain, or events bound that path at two or more different digests; or the record's `policy_digest` is not the live `TASKS.json` goalpost for the task it names. The message names which. An unreadable store is also a failure. In a contract with **no** event store the chain check is skipped — the mode's strength is store-dependent by construction (§17). | + +**Do not confuse `missing_evidence_record` with `missing_evidence_path`.** They +are opposite directions of the same pair. `missing_evidence_record` walks +*bundles* looking for the record that should describe them. +`missing_evidence_path` comes straight out of `verify_evidence()` and means a +*record's* `uri` does not resolve. Both can be true at once and neither implies +the other. **Evidence discovery (v0.11.0+).** `loop doctor` scans `.loop/evidence/*.json` when the directory exists; an absent directory with no runner bundle is a no-op that leaves every doctor key byte-identical (no new top-level key was added). A malformed or unparseable record **fails** doctor rather than being skipped, and `loop-engineer/evidence@1` joins `schemas_checked` when at least -one record was read. +one record was read. Since the evidence-wiring release each structurally-valid +record is additionally passed to `verify_evidence()` — surfacing +`hash_mismatch`, `missing_evidence_path`, `workspace_escape`, `not_a_file` and +`invalid_uri` **verbatim**, per the rule that doctor composes a verb and reuses +its issue codes rather than inventing parallel ones — and the latest record per +task is compared against the live goalpost (`policy_digest_mismatch`). The +`verify_evidence()` call is guarded on the record having no structural issues, +so a malformed record is reported once, not twice. The chain-binding walk is a +separate, store-gated check that lives beside `chain_columns_missing` in the +event-store block above; it costs one extra read-only fold of the store, because +it needs envelope-level `artifact_hashes` that no report returns and widening a +report's dict would leak into the `loop status` / `loop replay` CLI JSON. **`loop migrate`.** `loop migrate ` is the only store-upgrade path: explicit, idempotent, and non-rewriting. It widens `events` with the two diff --git a/reference/safety-and-approvals.md b/reference/safety-and-approvals.md index ad215f5..67d8640 100644 --- a/reference/safety-and-approvals.md +++ b/reference/safety-and-approvals.md @@ -98,6 +98,8 @@ This is why two structural rules from [[loop-repair]] are load-bearing safety ru Since v0.11.0 this invariant has a machine check: `loop doctor` reports `self_verified_evidence` when an evidence@1 record declares that its producer also verified it (`reference/repo-os-contract.md` §17). The check surfaces *declared* self-verification — a worker that writes a false verifier name, or that rewrites the record afterwards, is not caught, which is why the protected-file and canary rules above remain load-bearing. +The goalpost half of the invariant now has a machine check too: `loop doctor` reports `policy_digest_mismatch` when the latest evidence record for a task records a goalpost — `id` / `verify` / `criterion_ref` / `depends_on` — that is not the live `TASKS.json` entry, so moving the goalpost after a verification is a finding rather than a diff nobody reads. It binds the criterion *reference*, never the criterion *text*, and a rename still evades it (a record naming a task no longer in `TASKS.json` is not compared at all) — which is exactly why the protected-file rule and the canaries of §6 stay load-bearing rather than being replaced by it. + --- ## 6. Anti-cheat canaries diff --git a/schemas/evidence.schema.json b/schemas/evidence.schema.json index 18451ff..b861f92 100644 --- a/schemas/evidence.schema.json +++ b/schemas/evidence.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "loop-engineer/evidence@1", "title": "Loop Engineer Evidence @1", - "description": "A hashed evidence record with artifact provenance. loop doctor discovers and validates records from the declared location .loop/evidence/*.json; it does not yet hash-verify the artifacts they reference. See reference/repo-os-contract.md #17.", + "description": "A hashed evidence record with artifact provenance. loop doctor discovers records from the declared location .loop/evidence/*.json, validates them, hash-verifies the artifact each one references, and compares the latest record per task against the live TASKS.json goalpost; a verified dispatch also binds the record's bytes into the event hash chain. Hash verification proves the pointer, never the provenance, and code_digest is never re-hashed against the verifier file. See reference/repo-os-contract.md #17.", "type": "object", "required": ["schema", "id", "kind", "uri", "sha256", "media_type", "produced_by", "created_at"], "properties": { diff --git a/schemas/terminal.schema.json b/schemas/terminal.schema.json index db3ecb9..5a6e57c 100644 --- a/schemas/terminal.schema.json +++ b/schemas/terminal.schema.json @@ -77,7 +77,10 @@ ], "properties": { "mode": { - "const": "all_required" + "enum": [ + "all_required", + "all_required_verified_evidence" + ] } }, "additionalProperties": false, diff --git a/scripts/metrics.py b/scripts/metrics.py index 046d6bf..a8c6423 100644 --- a/scripts/metrics.py +++ b/scripts/metrics.py @@ -28,6 +28,13 @@ red gate, and a same-iteration green never excuses its own task's red — within-iteration chronology is unknowable, so it fails closed. +A verify bundle whose `verifier.source` is `injected_callable` is a caller's +self-report, not an independently-executed gate, so it is excluded from every +metric before FCR and RP are derived; the excluded filenames are listed under +`provenance.injected_verifier_bundles` so the exclusion is visible rather than a +silent FCR shift. A bundle carrying no `verifier` block at all has an unknown +source and still counts — grandfathering by absence, not by guess. + The `### Outcome` token contract: a claim counts toward the FCR denominator only when its outcome token is a recognized SUCCESS token — completion-class (`task_passed`, `terminal`, `succeeded`) or progress-class (`advanced`); @@ -75,6 +82,7 @@ if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) +from loop.evidence import verify_bundle_is_green # noqa: E402 from loop.paths import LoopPaths, resolve_loop_paths # noqa: E402 import re # noqa: E402 @@ -107,6 +115,11 @@ # invocation runs. evidence_backed via a verify script requires one to exist. _GATE_SCRIPTS = ("holdout_gate.py", "anticheat_scan.py", "anti_cheat.py") +# A verify bundle whose `verifier.source` is this ran a caller-supplied callable +# (`loop/verifier.py:111`), so its verdict is a self-report rather than the output +# of an independently-resolved command. It is never gate evidence. +_INJECTED_VERIFIER_SOURCE = "injected_callable" + _ITER_HEADER_RE = re.compile(r"(?m)^##\s+Iteration\s+(\S+)") # "outcome" declaration followed (within a little markup/whitespace) by its token. _OUTCOME_RE = re.compile(r"outcome[^A-Za-z0-9]{0,40}?([A-Za-z][A-Za-z_]{1,})", re.IGNORECASE | re.DOTALL) @@ -235,10 +248,12 @@ def _load_verify_bundles(loop_dir: Path) -> list[dict]: data = _read_json_object(path) if not data: continue - outcome = str(data.get("outcome", "")).upper() - green = outcome == "PASS" or data.get("passed") is True + # One definition, shared with the kernel's strict-completion bar. + green = verify_bundle_is_green(data) it = data.get("iteration_id", data.get("iteration")) task = data.get("task") + verifier = data.get("verifier") + source = verifier.get("source") if isinstance(verifier, dict) else None bundles.append( { "path": path, @@ -247,6 +262,7 @@ def _load_verify_bundles(loop_dir: Path) -> list[dict]: "iter": _norm_iter(it) if it is not None else None, "task": str(task) if task is not None else None, "score": _num(data.get("score")), + "source": source, } ) return bundles @@ -455,7 +471,16 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict terminal = _read_json_object(paths.terminal) blocks = _runlog_blocks(runlog_text) - bundles = _load_verify_bundles(loop_dir_path) + # An injected-callable verdict is the caller's own self-report, not an + # independently-executed gate (`reference/repo-os-contract.md` §17), so it is + # partitioned out before any metric consumes it. Only an EXPLICIT + # `injected_callable` source is excluded: a bundle with no `verifier` block has + # an unknown source and keeps counting — grandfathering by absence, not by + # guess. The excluded names are reported under provenance so the exclusion is + # visible rather than a silent FCR shift. + all_bundles = _load_verify_bundles(loop_dir_path) + injected = sorted(b["name"] for b in all_bundles if b["source"] == _INJECTED_VERIFIER_SOURCE) + bundles = [b for b in all_bundles if b["source"] != _INJECTED_VERIFIER_SOURCE] verify_by_iter, unmatched_verify = _assign_bundles_to_iters(bundles, blocks) # Success claims: RUNLOG success-outcome iterations, plus the terminal claim. @@ -600,6 +625,7 @@ def _claim_is_clean(iid: str) -> bool: "holdout_source": holdout_source, "holdout_verdicts": holdout_verdicts, "unmatched_verify": unmatched_verify, + "injected_verifier_bundles": injected, }, } diff --git a/scripts/test_adversarial_evidence_binding.py b/scripts/test_adversarial_evidence_binding.py new file mode 100644 index 0000000..4feeb78 --- /dev/null +++ b/scripts/test_adversarial_evidence_binding.py @@ -0,0 +1,540 @@ +"""What chain-bound evidence catches, and what it provably does not. + +The claim this file defends is "evidence whose bytes are bound into an ANCHORABLE +chain", never "tamper-proof evidence". Binding inherits Slice 1's trust model +exactly: a worker who can rewrite .loop/ can rewrite the store (repo-os-contract +#16, Integrity boundary), so without an external anchor a full rewrite verifies. + +Every ``*_pinned`` test states an honest limitation and carries the positive control +that bounds it: a pin whose detector was never shown to fire certifies a hole as +closed. The rest prove a real detection. +""" +from __future__ import annotations + +import hashlib +import json +import shutil +import sqlite3 +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from metrics import compute_metrics # noqa: E402 + +from chain_fixtures import drop_triggers, restore_triggers # noqa: E402 +from loop import emit # noqa: E402 +from loop.chain import compute_event_hash # noqa: E402 +from loop.completion import VERIFIED_EVIDENCE_MODE # noqa: E402 +from loop.contract import doctor_report # noqa: E402 +from loop.events import SQLiteEventStore # noqa: E402 +from loop.evidence import artifact_object_path # noqa: E402 +from loop.runner import dispatch_once # noqa: E402 +from loop.runtime import bound_artifact_digests # noqa: E402 +from loop.verifier import (executed_verifier_identity, # noqa: E402 + verification_policy_digest, verifier_code_digest) + +_EVENT_SCHEMA_ID = "loop-engineer/event@1" +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RUN_ID = "run-1" +_FORGED = '{"outcome": "PASS", "passed": true, "forged": true}\n' + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _task(task_id="T-1", **overrides): + base = {"id": task_id, "title": task_id, "status": "pending", "criterion_ref": task_id, + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None} + base.update(overrides) + return base + + +def _write_tasks(workspace, tasks): + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": list(tasks)}), encoding="utf-8") + + +def _contract(tmp_path, name="workspace", tasks=(), *, store=True): + """A contract, optionally with an event store ramped to execute-task. + + The ramp records the FSM walk, not work: its events carry iteration_id 0, so the + first dispatch is iteration 1 and every artifact path asserted below is literal. + Without a store the contract is the supported writer-API path — the one decision + 14 names as unable to chain-bind anything. + """ + workspace = tmp_path / name + emit.open_contract(workspace) + if tasks: + _write_tasks(workspace, tasks) + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + if store: + events = SQLiteEventStore(_store_path(workspace)) + events.append(_RUN_ID, "contract_opened", {"workspace": name}, actor="test") + for state in _RAMP: + events.append(_RUN_ID, "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + # The ramp is written straight to the store, so state.json still says intake. + # Doctor reconciles the two, so the fixture must land where the ramp landed. + state_path = workspace / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["state"] = _RAMP[-1] + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + return workspace + + +def _dispatched(tmp_path, name="workspace"): + workspace = _contract(tmp_path, name, [_task()]) + dispatch_once(workspace) + return workspace + + +def _store_path(workspace): + return workspace / ".loop" / "events.db" + + +def _bundle(workspace): + return workspace / ".loop" / "artifacts" / "verify-iter1.json" + + +def _record(workspace): + return workspace / ".loop" / "evidence" / "evidence-iter1.json" + + +def _head(workspace): + return doctor_report(workspace)["event_store"]["chain"]["head"]["event_hash"] + + +def _record_at(conn, sequence, prev_event_hash): + """Rebuild one row into the record dict read_event_rows projects (hash preimage shape).""" + row = conn.execute( + "SELECT run_id, sequence, event_id, type, actor, causation_id, correlation_id, ts, " + "payload, artifact_hashes FROM events WHERE sequence = ?", (sequence,)).fetchone() + return {"schema": _EVENT_SCHEMA_ID, "run_id": row[0], "sequence": row[1], "event_id": row[2], + "type": row[3], "actor": row[4], "causation_id": row[5], "correlation_id": row[6], + "ts": row[7], "payload": json.loads(row[8]), "artifact_hashes": json.loads(row[9]), + "prev_event_hash": prev_event_hash} + + +def _rechain(workspace, artifact_hashes): + """Replace the bound digests on the one binding event and re-chain from genesis.""" + store_path = _store_path(workspace) + drop_triggers(store_path) + conn = sqlite3.connect(str(store_path)) + try: + conn.execute("UPDATE events SET artifact_hashes = ? WHERE artifact_hashes != '[]'", + (json.dumps(artifact_hashes),)) + prev = None + for (sequence,) in conn.execute( + "SELECT sequence FROM events ORDER BY sequence ASC").fetchall(): + digest = compute_event_hash(_record_at(conn, sequence, prev)) + conn.execute("UPDATE events SET prev_event_hash = ?, event_hash = ? WHERE sequence = ?", + (prev, digest, sequence)) + prev = digest + conn.commit() + finally: + conn.close() + restore_triggers(store_path) + + +def _rewrite_everything(workspace, *, text=_FORGED): + """The competent adversary: new bundle bytes, a matching record, a matching object, + the bound row's artifact_hashes replaced, and the chain recomputed from genesis.""" + bundle, record_path = _bundle(workspace), _record(workspace) + record = json.loads(record_path.read_text(encoding="utf-8")) + artifact_object_path(workspace, record["sha256"]).unlink() # burn the recovery copy + bundle.write_text(text, encoding="utf-8") + new_sha = hashlib.sha256(text.encode("utf-8")).hexdigest() + record["sha256"] = new_sha + record_text = json.dumps(record, indent=2, sort_keys=True) + "\n" + record_path.write_text(record_text, encoding="utf-8") + obj = artifact_object_path(workspace, new_sha) + obj.parent.mkdir(parents=True, exist_ok=True) + obj.write_text(text, encoding="utf-8") + _rechain(workspace, [ + {"path": bundle.relative_to(workspace).as_posix(), "sha256": new_sha}, + {"path": record_path.relative_to(workspace).as_posix(), + "sha256": hashlib.sha256(record_text.encode("utf-8")).hexdigest()}, + {"path": obj.relative_to(workspace).as_posix(), "sha256": new_sha}, + ]) + return new_sha + + +def _unbind(workspace): + """Replay the stream into a fresh store with nothing bound — a pre-release run. + + The append-only triggers make retroactive UNbinding as impossible as retroactive + binding, so the only way a pre-binding contract can exist is the way it was + originally written: every event re-appended verbatim except its artifact_hashes. + """ + events = SQLiteEventStore(_store_path(workspace)).read(_RUN_ID) + for suffix in ("", "-wal", "-shm"): + _store_path(workspace).with_name("events.db" + suffix).unlink(missing_ok=True) + store = SQLiteEventStore(_store_path(workspace)) + for event in events: + store.append(_RUN_ID, event["type"], event["payload"], actor=event["actor"], + event_id=event["event_id"], causation_id=event["causation_id"], + correlation_id=event["correlation_id"], ts=event["ts"], + artifact_hashes=None) + + +def _handwritten_record(workspace, *, task_id="T-1"): + """A self-consistent, never-dispatched record: real bundle, real digest, real policy.""" + bundle = workspace / ".loop" / "artifacts" / "verify-iter9.json" + bundle.parent.mkdir(parents=True, exist_ok=True) + text = '{"outcome": "PASS", "passed": true}' + bundle.write_text(text, encoding="utf-8") + tasks = json.loads((workspace / "TASKS.json").read_text(encoding="utf-8")) + entry = next(t for t in tasks["tasks"] if t["id"] == task_id) + record = { + "schema": "loop-engineer/evidence@1", "id": "hand:9:verify", "kind": "verify-bundle", + "uri": ".loop/artifacts/verify-iter9.json", + "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), + "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", + "produced_by": {"run_id": _RUN_ID, "task_id": task_id, "attempt": 1, + "executor": "worker-a"}, + "verified_by": {"by": "ci", "at": "2026-07-25T00:00:00+00:00", + "command": entry["verify"], "code_digest": None, + "code_digest_basis": "path_lookup", + "policy_digest": verification_policy_digest(entry)}, + } + path = workspace / ".loop" / "evidence" / "evidence-iter9.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return ".loop/evidence/evidence-iter9.json" + + +def _terminal_file(workspace, *, mode, evidence): + (workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "project": workspace.name, "state": "Succeeded", + "criteria_met": {"T-1": True}, "completion_policy": {"mode": mode}, + "evidence": list(evidence), "false_completion": False, + "terminated_at": "2026-07-25T00:00:00+00:00", "reason": "hand-written", + }, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _metrics_workspace(tmp_path, source): + """A minimal metrics-readable loop dir: one green claim backed by one green bundle.""" + target = tmp_path / "metrics-ws" + (target / ".loop" / "artifacts").mkdir(parents=True, exist_ok=True) + (target / "RUNLOG.md").write_text( + "# RUNLOG\n\n## Iteration 1 — 2026-07-25\n\n### Outcome\n\n`task_passed`\n", + encoding="utf-8") + (target / ".loop" / "state.json").write_text(json.dumps( + {"schema": "loop-engineer/state@1", "state": "execute-task", "iteration_id": 1, + "terminal_state": None}), encoding="utf-8") + (target / ".loop" / "artifacts" / "verify-iter1.json").write_text(json.dumps( + {"iteration_id": 1, "task": "T-1", "outcome": "PASS", "passed": True, "score": 1.0, + "verifier": {"by": "loop.run", "source": source}}), encoding="utf-8") + return target + + +# --- pinned honest limitations ------------------------------------------------- + +def test_a_full_rewrite_of_artifacts_and_store_is_not_caught_without_an_anchor_pinned(tmp_path): + """Binding proves the bytes AGREE, never that a dispatch happened. + + A worker who can rewrite .loop/ owns the store too: rewrite the bundle, recompute + the record and the content-addressed object, replace the digests the event bound and + re-chain from genesis, and every internal check agrees again. This is Slice 1's + trust model unchanged (repo-os-contract #16) — which is why the shipped claim is + "bound into an ANCHORABLE chain" and never "tamper-proof". The anchor that does + catch it is the very next test. + """ + workspace = _dispatched(tmp_path) + assert doctor_report(workspace)["ok"] is True # control: clean before + before = bound_artifact_digests(workspace)[".loop/artifacts/verify-iter1.json"] + new_sha = _rewrite_everything(workspace) + # the forge is real, not a no-op: different bytes, and the chain now commits to them + assert new_sha != before + # One digest, not a conflict set: the re-chain REPLACES the binding rather than + # appending a second one, which is exactly why this rewrite still verifies clean. + assert bound_artifact_digests(workspace)[".loop/artifacts/verify-iter1.json"] == (new_sha,) + assert json.loads(_bundle(workspace).read_text(encoding="utf-8"))["forged"] is True + report = doctor_report(workspace) + assert report["ok"] is True and report["issues"] == [] + + +def test_a_record_written_outside_a_dispatch_is_never_chain_bound_pinned(tmp_path): + """``emit.write_verify_evidence`` is a WRITER, not a dispatch: it appends no event. + + Nothing binds what it wrote, so a later rewrite of that record is invisible. The + writer-API path is supported deliberately (decision 14's residual (i)), so this + silence ships NAMED rather than closed. Positive control: the identical rewrite in a + dispatched contract is caught, which is the only thing that makes this a boundary + rather than an absence of checking. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + written = emit.write_verify_evidence( + workspace, run_id=_RUN_ID, iteration_id=1, task=_task(), passed=True, + code_identity=executed_verifier_identity(_VERIFY, workspace)) + assert bound_artifact_digests(workspace) is None # no store: nothing can bind + record = json.loads(written["evidence"].read_text(encoding="utf-8")) + record["verified_by"]["by"] = "ci" + written["evidence"].write_text(json.dumps(record), encoding="utf-8") + assert "evidence_chain_mismatch" not in _codes(doctor_report(workspace)) + + bound = _dispatched(tmp_path, "bound") # control: a dispatch DOES bind + data = json.loads(_record(bound).read_text(encoding="utf-8")) + data["verified_by"]["by"] = "ci" + _record(bound).write_text(json.dumps(data), encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(bound)) + + +def test_a_legacy_iteration_event_can_never_be_bound_retroactively_pinned(tmp_path): + """A pre-release event carries ``artifact_hashes: []`` and is silent by construction. + + It cannot be repaired either: the append-only triggers forbid the UPDATE a backfill + would need — the same reason ``loop migrate`` refuses to backfill chain hashes. So a + pre-release iteration whose evidence was deleted stays undetectable at this layer + forever. Positive control: the same deletion over a BOUND event is reported. + """ + workspace = _dispatched(tmp_path) + _unbind(workspace) + assert bound_artifact_digests(workspace) == {} + _record(workspace).unlink() + _bundle(workspace).unlink() + assert "missing_bound_evidence" not in _codes(doctor_report(workspace)) + with pytest.raises((sqlite3.IntegrityError, sqlite3.OperationalError), match="append-only"): + conn = sqlite3.connect(str(_store_path(workspace))) + try: + conn.execute("UPDATE events SET artifact_hashes = '[]' WHERE sequence = 1") + finally: + conn.close() + + bound = _dispatched(tmp_path, "bound") # control: a bound pair IS missed + _record(bound).unlink() + _bundle(bound).unlink() + assert "missing_bound_evidence" in _codes(doctor_report(bound)) + + +def test_doctor_does_not_re_hash_the_verifier_file_pinned(tmp_path): + """``code_digest`` stays in the recorded-not-compared tier (decision 5). + + A verifier file legitimately changes between runs, and a comparison with no declared + baseline would fire on every honest edit. Only what the chain BOUND is re-hashed, and + the verifier script is not a bound artifact — so editing it after the dispatch leaves + doctor clean. Control: the bytes demonstrably moved, and the record still says + otherwise. + """ + workspace = _dispatched(tmp_path) + recorded = json.loads(_record(workspace).read_text(encoding="utf-8"))["verified_by"]["code_digest"] + script = workspace / "scripts" / "verify-fast.sh" + script.write_text("#!/bin/sh\nexit 0\n# tampered\n", encoding="utf-8") + assert verifier_code_digest(_VERIFY, workspace)[0] != recorded # control: bytes moved + assert "scripts/verify-fast.sh" not in bound_artifact_digests(workspace) + assert doctor_report(workspace)["ok"] is True + + +def test_an_older_record_for_a_moved_goalpost_is_not_compared_pinned(tmp_path): + """Decision 5's cost, stated plainly: only the LATEST record per task is compared. + + A superseded record therefore keeps a goalpost nobody checks. The alternative — + comparing every record — turns each honest re-verification into a permanent doctor + failure, because the older record legitimately describes the goalpost that was + current when it was written. Positive control: move the goalpost the LATEST record + cites and the comparison fires. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + identity = executed_verifier_identity(_VERIFY, workspace) + emit.write_verify_evidence(workspace, run_id=_RUN_ID, iteration_id=1, task=_task(), + passed=False, code_identity=identity) + emit.write_verify_evidence(workspace, run_id=_RUN_ID, iteration_id=2, + task=_task(verify="true"), passed=True, code_identity=identity) + _write_tasks(workspace, [_task(verify="true")]) # the LATEST record's goalpost + stale = json.loads(_record(workspace).read_text(encoding="utf-8"))["verified_by"]["policy_digest"] + assert stale != verification_policy_digest(_task(verify="true")) # iter1 IS stale + assert "policy_digest_mismatch" not in _codes(doctor_report(workspace)) + + _write_tasks(workspace, [_task(verify="false")]) # control: move the LATEST goalpost + assert "policy_digest_mismatch" in _codes(doctor_report(workspace)) + + +def test_deleting_every_artifact_and_the_store_leaves_a_clean_doctor_pinned(tmp_path): + """Delete the whole run and the contract is one that never ran. + + Artifacts, records, store AND sidecars gone: nothing inside the tree remembers the + dispatch, so doctor is clean and correct to be (repo-os-contract §22 — the + ``missing_event_store`` tripwire fires only on the sloppier deletion that leaves + -wal/-shm behind). Detection needs something OUTSIDE the tree; the anchored control + at the end of this test is that something. + """ + workspace = _dispatched(tmp_path) + anchor = _head(workspace) + shutil.rmtree(workspace / ".loop" / "artifacts") + shutil.rmtree(workspace / ".loop" / "evidence") + for suffix in ("", "-wal", "-shm"): + _store_path(workspace).with_name("events.db" + suffix).unlink(missing_ok=True) + report = doctor_report(workspace) + assert report["ok"] is True and report["issues"] == [] + assert "chain_anchor_mismatch" in _codes(doctor_report(workspace, expect_chain_head=anchor)) + + +def test_the_verified_evidence_mode_is_opt_in_and_not_retroactive_pinned(tmp_path): + """A Succeeded terminal written under ``all_required`` keeps the OLD bar forever. + + Non-empty path strings remain sufficient there, which is why both shipped examples + and every pre-release run survive this slice unchanged (Global Constraints: the + tightening is opt-in, never retroactive). The positive control is exactly one field: + the byte-identical terminal under the strict mode is reported. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + _terminal_file(workspace, mode="all_required", evidence=["RUNLOG.md"]) + assert "unverified_evidence_terminal" not in _codes(doctor_report(workspace)) + _terminal_file(workspace, mode=VERIFIED_EVIDENCE_MODE, evidence=["RUNLOG.md"]) + assert "unverified_evidence_terminal" in _codes(doctor_report(workspace)) + + +def test_a_hand_written_record_pointing_at_a_real_file_still_passes_pinned(tmp_path): + """Hash verification proves the POINTER, never the PROVENANCE. + + A record hand-written to describe a file that exists is, to doctor, indistinguishable + from one a dispatch produced: same shape, same true digest, same live goalpost. What + this release added is that the pointer must be TRUE — the control below corrupts the + pointed-at bytes and the new check fires. + """ + workspace = _contract(tmp_path, "storeless", [_task()], store=False) + _handwritten_record(workspace) + assert doctor_report(workspace)["ok"] is True + (workspace / ".loop" / "artifacts" / "verify-iter9.json").write_text("{}", encoding="utf-8") + assert "hash_mismatch" in _codes(doctor_report(workspace)) # control: it fires + + +def test_criterion_text_is_still_unbound_pinned(tmp_path): + """What a criterion MEANS is prose, and prose is bound by nothing. + + ``policy_digest`` covers id / verify / criterion_ref / depends_on — the POINTER to a + criterion, not its wording — so rewriting SPEC.md's acceptance text moves no digest + and fires nothing. Binding evidence does not bind intent. Control: the bundle IS a + bound artifact and SPEC.md is not, so the difference is structural, not incidental. + """ + workspace = _dispatched(tmp_path) + recorded = json.loads(_record(workspace).read_text(encoding="utf-8"))["verified_by"]["policy_digest"] + live = json.loads((workspace / "TASKS.json").read_text(encoding="utf-8"))["tasks"][0] + spec = workspace / "SPEC.md" + spec.write_text(spec.read_text(encoding="utf-8").replace( + "REPLACE: first success criterion", "anything at all now counts as success"), + encoding="utf-8") + assert "anything at all" in spec.read_text(encoding="utf-8") # the goal moved + assert verification_policy_digest(live) == recorded # the digest did not + bound = bound_artifact_digests(workspace) + assert "SPEC.md" not in bound and ".loop/artifacts/verify-iter1.json" in bound + assert doctor_report(workspace)["ok"] is True + + +def test_metrics_still_counts_a_hand_written_declared_command_bundle_pinned(tmp_path): + """``verifier.source`` is a string the WRITER declares, not a fact metrics can check. + + Hand-writing ``declared_command`` over ``injected_callable`` restores the bundle's + gate-evidence status and moves FCR back to 0.0. The exclusion is an honesty aid for + honest writers, never a control against a dishonest one. Control: the same bundle, + one string different, scores the opposite way. + """ + loop_dir = _metrics_workspace(tmp_path, "injected_callable") + assert compute_metrics(loop_dir)["false_completion_rate"] == 1.0 # control: excluded + bundle = loop_dir / ".loop" / "artifacts" / "verify-iter1.json" + body = json.loads(bundle.read_text(encoding="utf-8")) + body["verifier"]["source"] = "declared_command" + bundle.write_text(json.dumps(body), encoding="utf-8") + metrics = compute_metrics(loop_dir) + assert metrics["false_completion_rate"] == 0.0 + assert metrics["provenance"]["injected_verifier_bundles"] == [] + + +def test_the_strict_mode_accepts_a_hand_written_record_in_a_store_less_contract_pinned(tmp_path): + """Decision 14's residual (i), stated as MEASURED rather than as hoped. + + Where no event store exists there is nothing to bind against, so the strict mode + degrades to hash-verification plus goalpost agreement — and a hand-written record + satisfies it with zero dispatch and zero events. + + The plan claimed doctor's ``missing_event_store`` tripwire is what catches deleting a + store out from under a real run. Measured here, it is NOT: that tripwire is gated on + leftover -wal/-shm sidecars, and sidecar-free reads (v0.10.0) leave none, so deleting + the store makes plain doctor CLEANER — ok, zero issues — rather than louder. The real + compensating control is the EXTERNAL ANCHOR, asserted below; a pin that named the + tripwire instead would be a false claim shipped inside an honesty test. + """ + storeless = _contract(tmp_path, "storeless", [_task()], store=False) + entry = _handwritten_record(storeless) + path = emit.terminate(storeless, state="Succeeded", criteria_met={"T-1": True}, + evidence=[entry], completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} # ACCEPTED — residual + + bound = _dispatched(tmp_path, "bound") # control: a store refuses it + with pytest.raises(emit.EmitError, match="not bound"): + emit.terminate(bound, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_handwritten_record(bound)], + completion_policy=VERIFIED_EVIDENCE_MODE) + assert "chain_anchor_mismatch" not in _codes( + doctor_report(bound, expect_chain_head=_head(bound))) # anchor is quiet when intact + + deleted = _dispatched(tmp_path, "deleted") # the delete-the-store route + anchor = _head(deleted) + for suffix in ("", "-wal", "-shm"): + _store_path(deleted).with_name("events.db" + suffix).unlink(missing_ok=True) + silent = doctor_report(deleted) + assert silent["ok"] is True and silent["issues"] == [] # MEASURED: deleting it is silent + assert bound_artifact_digests(deleted) is None + emit.terminate(deleted, state="Succeeded", criteria_met={"T-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) # ACCEPTED — residual + assert "chain_anchor_mismatch" in _codes( + doctor_report(deleted, expect_chain_head=anchor)) # the anchor catches it + + +def test_a_full_rewrite_satisfies_the_strict_mode_without_an_anchor_pinned(tmp_path): + """The headline residual restated at the COMPLETION layer. + + Rewrite the bundle, the record, the object and the digests the event bound, re-chain + the tail, and ``terminate(..., all_required_verified_evidence)`` succeeds: the strict + mode is exactly as strong as the chain it reads, and the chain is as strong as its + anchor. Positive control: the same tree, checked against the pre-rewrite head, reports + ``chain_anchor_mismatch``. This pin is why the shipped claim stays "bound into an + anchorable chain" and why "tamper-proof evidence" is forbidden. + """ + workspace = _dispatched(tmp_path) + anchor = _head(workspace) + _rewrite_everything(workspace) + path = emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} + assert json.loads(_bundle(workspace).read_text(encoding="utf-8"))["forged"] is True + assert "chain_anchor_mismatch" in _codes( + doctor_report(workspace, expect_chain_head=anchor)) # control: the anchor + + +# --- real detections ----------------------------------------------------------- + +def test_an_anchored_head_catches_the_full_rewrite(tmp_path): + """The positive control for the first pin: an anchor recorded outside the tree.""" + workspace = _dispatched(tmp_path) + anchor = _head(workspace) + assert "chain_anchor_mismatch" not in _codes( + doctor_report(workspace, expect_chain_head=anchor)) + _rewrite_everything(workspace) + assert "chain_anchor_mismatch" in _codes( + doctor_report(workspace, expect_chain_head=anchor)) + + +def test_deleting_the_object_alone_is_reported(tmp_path): + """The content-addressed object is a BOUND artifact, not a convenience copy. + + Deleting the recovery source an ``evidence_chain_mismatch`` message points at is + itself detectable, so the recovery path cannot be quietly removed first. + """ + workspace = _dispatched(tmp_path) + digest = json.loads(_record(workspace).read_text(encoding="utf-8"))["sha256"] + obj = artifact_object_path(workspace, digest) + assert obj.is_file() + obj.unlink() + assert "missing_bound_evidence" in _codes(doctor_report(workspace)) diff --git a/scripts/test_adversarial_verifier_identity.py b/scripts/test_adversarial_verifier_identity.py index 03780df..f52b62d 100644 --- a/scripts/test_adversarial_verifier_identity.py +++ b/scripts/test_adversarial_verifier_identity.py @@ -9,12 +9,18 @@ import hashlib import json +import pytest + from loop import emit from loop.contract import doctor_report +from loop.events import SQLiteEventStore from loop.evidence import verify_evidence +from loop.runner import dispatch_once from loop.verifier import (executed_verifier_identity, injected_verifier_identity, verification_policy_digest, verifier_code_digest) +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + def _task(**overrides): base = {"id": "T-1", "title": "t", "status": "pending", "criterion_ref": "C-1", @@ -32,6 +38,37 @@ def _ws(tmp_path): return workspace +@pytest.fixture +def bound_workspace(tmp_path): + """One real dispatch behind an event store: iteration 1's artifacts ARE chain-bound. + + The store's ramp records the FSM walk to execute-task, not work — its events carry + iteration_id 0 — so the first dispatch is iteration 1 and both paths are literal. + """ + workspace = tmp_path / "bound" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [_task(criterion_ref="T-1", title="T-1")]}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "bound"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + state_path = workspace / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["state"] = _RAMP[-1] + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + dispatch_once(workspace) + return {"workspace": workspace, + "bundle": workspace / ".loop" / "artifacts" / "verify-iter1.json", + "record": workspace / ".loop" / "evidence" / "evidence-iter1.json"} + + def _write(workspace, *, iteration_id=1, task=None, passed=True, **kwargs): task = task or _task() kwargs.setdefault("code_identity", @@ -64,10 +101,15 @@ def test_unattributed_executor_never_trips_the_finding_pinned(tmp_path): assert "self_verified_evidence" not in _codes(doctor_report(workspace)) -def test_deleting_both_bundle_and_record_leaves_doctor_clean_pinned(tmp_path): - """Deleting the RECORD alone is caught (missing_evidence_record). Deleting the - PAIR is not: nothing outside the event log remembers the dispatch happened. - Slice 3 closes this by making Succeeded require hash-verified evidence. +def test_deleting_both_bundle_and_record_is_caught_only_when_the_chain_binds_them(tmp_path, + bound_workspace): + """Was `test_deleting_both_bundle_and_record_leaves_doctor_clean_pinned`; the + detection half arrived with chain-bound evidence in the evidence-wiring release. + + The RESIDUAL is unchanged and kept: written through the writer API with no event + store behind it, nothing outside the tree remembers the dispatch happened, so + deleting the pair is still invisible. Where a dispatch bound them, the committed + event names artifacts that are gone — `missing_bound_evidence`. """ workspace = _ws(tmp_path) written = _write(workspace, executor="w", verifier_identity="w") @@ -75,13 +117,21 @@ def test_deleting_both_bundle_and_record_leaves_doctor_clean_pinned(tmp_path): written["evidence"].unlink() assert "missing_evidence_record" in _codes(doctor_report(workspace)) # residue IS caught written["bundle"].unlink() - assert doctor_report(workspace)["ok"] is True # the pair is not + assert doctor_report(workspace)["ok"] is True # residual: the pair is not + bound_workspace["record"].unlink() + bound_workspace["bundle"].unlink() + assert "missing_bound_evidence" in _codes(doctor_report(bound_workspace["workspace"])) + +def test_rewriting_the_evidence_record_is_caught_only_when_the_chain_binds_it(tmp_path, + bound_workspace): + """Was `test_rewriting_the_evidence_record_to_independent_identities_leaves_doctor_clean_pinned`; + the detection half arrived with chain-bound evidence in the evidence-wiring release. -def test_rewriting_the_evidence_record_to_independent_identities_leaves_doctor_clean_pinned(tmp_path): - """The record is a plain file OUTSIDE the hash chain. A worker with write access - to .loop/ can rewrite it after the fact and doctor will not notice. Binding the - record digest into the chain is Slice 3. + The RESIDUAL is unchanged and kept: a record written by the writer API with no event + store behind it is still a plain file, and a rewrite of it is still invisible. A + record a dispatch bound is not — its bytes are covered by `event_hash`, so the + rewrite surfaces as `evidence_chain_mismatch`. """ workspace = _ws(tmp_path) written = _write(workspace, executor="solo", verifier_identity="solo") @@ -89,12 +139,23 @@ def test_rewriting_the_evidence_record_to_independent_identities_leaves_doctor_c record = json.loads(written["evidence"].read_text(encoding="utf-8")) record["verified_by"]["by"] = "ci" written["evidence"].write_text(json.dumps(record), encoding="utf-8") - assert doctor_report(workspace)["ok"] is True + assert doctor_report(workspace)["ok"] is True # residual + bound = bound_workspace["record"] + data = json.loads(bound.read_text(encoding="utf-8")) + data["verified_by"]["by"] = "ci" + bound.write_text(json.dumps(data), encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(bound_workspace["workspace"])) def test_hand_written_record_with_a_fabricated_code_digest_is_doctor_clean_pinned(tmp_path): """Digests are values the WRITER asserts. Doctor validates their shape, never their truth — a hand-written record is indistinguishable from a runner-written one. + + Narrower since the evidence-wiring release, and deliberately still true: what IS + now checked is the bundle's BYTES (the record's `sha256` must hash the file its + `uri` names) and, for a task still declared in TASKS.json, the recorded + `policy_digest` against the live goalpost. `code_digest` remains unchecked, and + this record names a task the scaffold's TASKS.json does not declare. """ workspace = _ws(tmp_path) written = _write(workspace) @@ -108,29 +169,44 @@ def test_hand_written_record_with_a_fabricated_code_digest_is_doctor_clean_pinne assert doctor_report(workspace)["ok"] is True -def test_no_automated_digest_comparison_exists_pinned(tmp_path): - """Two records for the same task with DIFFERENT policy digests: doctor is silent. - Nothing in this release compares a recorded digest against anything. +def test_policy_digest_comparison_fires_only_against_a_live_task_entry(tmp_path): + """Was `test_no_automated_digest_comparison_exists_pinned`; the comparison arrived in + the evidence-wiring release. + + The RESIDUAL is its two silences, both kept here: a record naming a task TASKS.json + no longer declares is never compared (a rename or removal is a replan, not a + forgery), and neither is any record that is not the latest for its task. Both + records below move the goalpost, and doctor stays quiet until the task is live. """ workspace = _ws(tmp_path) - _write(workspace, iteration_id=1, task=_task()) + _write(workspace, iteration_id=1, task=_task()) # T-1 is not in TASKS.json _write(workspace, iteration_id=2, task=_task(verify="true")) digests = {json.loads((workspace / ".loop" / "evidence" / f"evidence-iter{n}.json") .read_text(encoding="utf-8"))["verified_by"]["policy_digest"] for n in (1, 2)} - assert len(digests) == 2 # the goalpost demonstrably moved - assert doctor_report(workspace)["ok"] is True # and nothing surfaced it + assert len(digests) == 2 # the goalpost demonstrably moved + assert "policy_digest_mismatch" not in _codes(doctor_report(workspace)) # residual + (workspace / "TASKS.json").write_text(json.dumps( + {"schema": "loop-engineer/tasks@1", "project": "p", "tasks": [_task(status="pending")]}), + encoding="utf-8") + assert "policy_digest_mismatch" in _codes(doctor_report(workspace)) + +def test_doctor_hash_verifies_the_referenced_bundle(tmp_path): + """Was `test_doctor_does_not_hash_verify_the_referenced_bundle_pinned`; doctor gained + the composition in the evidence-wiring release. -def test_doctor_does_not_hash_verify_the_referenced_bundle_pinned(tmp_path): - """Slice 2 checks structure, declared independence, and record presence only.""" + Slice 2 checked structure, declared independence and record presence only, and the + hash check existed but nothing called it. Doctor now composes `verify_evidence` over + every discovered record and surfaces its `hash_mismatch` verbatim. + """ workspace = _ws(tmp_path) written = _write(workspace, passed=False) written["bundle"].write_text(json.dumps({"outcome": "PASS", "passed": True}), encoding="utf-8") record = json.loads(written["evidence"].read_text(encoding="utf-8")) - # control: the hash check EXISTS and catches this swap — doctor just never calls it. + # the standalone verb and the doctor sweep now agree, and doctor says it in its issues assert verify_evidence(record, workspace_root=workspace)["ok"] is False - assert doctor_report(workspace)["ok"] is True + assert "hash_mismatch" in _codes(doctor_report(workspace)) def test_code_digest_is_null_for_the_common_python_m_pytest_command_pinned(tmp_path): diff --git a/scripts/test_bound_artifact_view.py b/scripts/test_bound_artifact_view.py new file mode 100644 index 0000000..6e33bc5 --- /dev/null +++ b/scripts/test_bound_artifact_view.py @@ -0,0 +1,233 @@ +"""The write-time and read-time views of "bound" must agree on every tree. + +Two defects lived in the gap between them. + +**The append-only forge.** ``bound_artifact_digests`` built a ``{path: sha256}`` dict +comprehension over every event, so a path bound twice collapsed LAST-WINS. No trigger +drop and no re-chain were needed: rewrite a bound record (``emit.terminate`` correctly +refuses — "bound at a different digest"), then append ONE ordinary event binding that +path at the NEW digest, and the writer's view said "bound, and it matches" while +``_bound_evidence_issues`` — which checks PER EVENT — still reported +``evidence_chain_mismatch`` on the same tree. A path carrying two or more different +digests is ambiguous, and ambiguous is not proof, so the view now returns the whole +conflict set and the strict bar refuses it. + +**The unbounded read of an attacker-named path.** ``event@1`` constrains a bound +``path`` only to ``minLength: 1``, and the walk joined it straight onto the workspace, +so an appended event could make ``loop doctor`` open and hash ``/etc/hostname``. Every +other workspace-bearing layer containment-checks; this one now does too, BEFORE it +opens anything, and reads under a cap. + +``bound_artifact_digests`` shipped with no direct test at all (Task-3 carry-forward); +it has one now. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from loop import emit +from loop.completion import VERIFIED_EVIDENCE_MODE +from loop.contract import doctor_report +from loop.evidence import hash_bound_artifact +from loop.events import SQLiteEventStore +from loop.runner import dispatch_once +from loop.runtime import RuntimeStoreError, bound_artifact_digests +from loop.scaffold import scaffold + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RUN_ID = "run-1" +_RECORD = ".loop/evidence/evidence-iter1.json" + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _store(workspace): + return SQLiteEventStore(workspace / ".loop" / "events.db") + + +def _dispatched(tmp_path, name="workspace"): + workspace = tmp_path / name + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [{"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "T-1", + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None}], + }), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = _store(workspace) + store.append(_RUN_ID, "contract_opened", {"workspace": name}, actor="test") + for state in _RAMP: + store.append(_RUN_ID, "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + dispatch_once(workspace) + return workspace + + +def _append_binding(workspace, artifact_hashes, *, iteration_id=2): + _store(workspace).append( + _RUN_ID, "iteration_appended", + {"iteration_id": iteration_id, "outcome": "task_passed", "state": "execute-task"}, + actor="forge", artifact_hashes=artifact_hashes) + + +# --- the append-only forge ---------------------------------------------------- + + +def _forge(workspace): + """Rewrite the bound record, then re-bind it at its new digest by APPENDING.""" + record_path = workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + record["verified_by"]["by"] = "forged-ci" + text = json.dumps(record, indent=2, sort_keys=True) + "\n" + record_path.write_text(text, encoding="utf-8") + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def test_an_append_only_forge_cannot_launder_a_tampered_record(tmp_path): + workspace = _dispatched(tmp_path) + digest = _forge(workspace) + # Control: before the re-bind BOTH layers already refuse, for the same reason. + with pytest.raises(emit.EmitError, match="bound at a different digest"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + assert "evidence_chain_mismatch" in _codes(doctor_report(workspace)) + + _append_binding(workspace, [{"path": _RECORD, "sha256": digest}]) + # The read-time walk still reports the FIRST binding's mismatch... + assert "evidence_chain_mismatch" in _codes(doctor_report(workspace)) + # ...and the write-time view no longer disagrees with it. + with pytest.raises(emit.EmitError, match="ambiguous binding is not proof"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +# --- bound_artifact_digests, directly ----------------------------------------- + + +def test_no_store_returns_none_and_nothing_bound_returns_an_empty_dict(tmp_path): + scaffolded = tmp_path / "scaffolded" + scaffold(scaffolded) + assert bound_artifact_digests(scaffolded) is None + bare = tmp_path / "bare" + emit.open_contract(bare) + _store(bare).append(_RUN_ID, "contract_opened", {"workspace": "bare"}, actor="test") + assert bound_artifact_digests(bare) == {} + + +def test_a_dispatch_binds_three_paths_each_at_exactly_one_digest(tmp_path): + workspace = _dispatched(tmp_path) + bound = bound_artifact_digests(workspace) + assert set(bound) == {".loop/artifacts/verify-iter1.json", _RECORD, + *(p for p in bound if p.startswith(".loop/artifacts/objects/"))} + assert all(len(digests) == 1 for digests in bound.values()) + + +def test_repeat_bindings_at_the_same_digest_stay_one_element(tmp_path): + workspace = _dispatched(tmp_path) + same = bound_artifact_digests(workspace)[_RECORD][0] + _append_binding(workspace, [{"path": _RECORD, "sha256": same}]) + assert bound_artifact_digests(workspace)[_RECORD] == (same,) + + +def test_conflicting_bindings_are_returned_in_first_bound_order(tmp_path): + workspace = _dispatched(tmp_path) + first = bound_artifact_digests(workspace)[_RECORD][0] + second, third = "a" * 64, "b" * 64 + _append_binding(workspace, [{"path": _RECORD, "sha256": second}], iteration_id=2) + _append_binding(workspace, [{"path": _RECORD, "sha256": third}], iteration_id=3) + assert bound_artifact_digests(workspace)[_RECORD] == (first, second, third) + + +def test_an_unreadable_store_raises_rather_than_returning_an_empty_view(tmp_path): + workspace = _dispatched(tmp_path) + for sidecar in ("events.db-wal", "events.db-shm"): + (workspace / ".loop" / sidecar).unlink(missing_ok=True) + (workspace / ".loop" / "events.db").write_bytes(b"not a sqlite database" * 64) + with pytest.raises(RuntimeStoreError): + bound_artifact_digests(workspace) + + +# --- containment in the binding walk ------------------------------------------ + + +@pytest.mark.parametrize("escaping", [ + "/etc/hostname", + "../../../../etc/hostname", + "nested/../../outside.txt", + "C:/Windows/system.ini", + "..\\..\\etc\\hostname", + " ", +]) +def test_an_escaping_bound_path_is_reported_not_read(tmp_path, escaping): + workspace = _dispatched(tmp_path) + _append_binding(workspace, [{"path": escaping, "sha256": "0" * 64}]) + report = doctor_report(workspace) + assert "bound_evidence_escape" in _codes(report) + assert report["ok"] is False + + +def test_the_lexical_containment_check_touches_no_filesystem(tmp_path, monkeypatch): + """Proof, not assertion: with ``os.open`` and ``Path.resolve`` booby-trapped, an + escaping path still returns a verdict — so nothing was opened or even stat'd.""" + def boom(*args, **kwargs): # pragma: no cover - must never run + raise AssertionError("the walk touched the filesystem before containment") + + monkeypatch.setattr("loop.evidence.os.open", boom) + monkeypatch.setattr(Path, "resolve", boom) + code, detail = hash_bound_artifact(tmp_path, "../../../../etc/hostname") + assert code == "escape" and ".." in detail + + +@pytest.mark.parametrize("rel", ["", " ", None, 7]) +def test_a_path_that_is_not_a_path_is_an_escape_not_a_crash(tmp_path, rel): + """``event@1``'s ``minLength: 1`` keeps the empty string out of a real store, so the + helper's own total-ness is what stops a hand-built or future store from crashing the + walk instead of reporting it.""" + code, detail = hash_bound_artifact(tmp_path, rel) + assert code == "escape" and "non-empty path" in detail + + +def test_a_symlinked_escape_is_caught_after_resolution(tmp_path): + workspace = tmp_path / "ws" + workspace.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("secret", encoding="utf-8") + (workspace / "link.txt").symlink_to(outside) + code, detail = hash_bound_artifact(workspace, "link.txt") + assert code == "escape" and "outside the workspace" in detail + + +def test_a_contained_regular_file_hashes(tmp_path): + (tmp_path / "a.txt").write_text("hello", encoding="utf-8") + assert hash_bound_artifact(tmp_path, "a.txt") == ( + "ok", hashlib.sha256(b"hello").hexdigest()) + + +def test_a_non_regular_file_is_unreadable_rather_than_read_forever(tmp_path): + os.mkfifo(tmp_path / "pipe") + code, detail = hash_bound_artifact(tmp_path, "pipe") + assert code == "unreadable" and "not a regular file" in detail + + +def test_the_read_is_capped(tmp_path): + (tmp_path / "big.bin").write_bytes(b"x" * 4096) + code, detail = hash_bound_artifact(tmp_path, "big.bin", max_bytes=16) + assert code == "unreadable" and "read cap" in detail + + +def test_an_absent_bound_path_is_still_missing_bound_evidence(tmp_path): + """The pre-existing code is unchanged for the pre-existing case.""" + workspace = _dispatched(tmp_path) + (workspace / ".loop" / "evidence" / "evidence-iter1.json").unlink() + assert "missing_bound_evidence" in _codes(doctor_report(workspace)) diff --git a/scripts/test_completion_verdict.py b/scripts/test_completion_verdict.py new file mode 100644 index 0000000..773307d --- /dev/null +++ b/scripts/test_completion_verdict.py @@ -0,0 +1,258 @@ +"""The strict mode must cite evidence of PASSING, not merely authentic evidence. + +Every other sub-check of ``all_required_verified_evidence`` asks whether the record is +real: it validates, its pointer resolves and hashes, some event bound those bytes, its +goalpost is live. None of them asked what the verdict SAYS — so a dispatch whose +verifier FAILED produced a perfectly authentic, chain-bound record that backed +``Succeeded`` end to end: ``emit.terminate`` accepted it, ``loop doctor`` reported +``ok: true`` with zero issues, and the reducer replayed ``Succeeded``. + +The verdict is judged by the repo's ONE green-marker rule +(``loop.evidence.verify_bundle_is_green``), which ``scripts/metrics.py`` now imports +rather than restating: a bundle is green when ``outcome == "PASS"`` or +``passed is True``, and a ``score``-only bundle reads RED. A bundle that reads RED to +the metrics gate can therefore never read GREEN to the completion gate. +""" +from __future__ import annotations + +import hashlib +import json + +import metrics +import pytest + +from loop import emit, evidence +from loop.completion import VERIFIED_EVIDENCE_MODE +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.reducer import reduce_events +from loop.runner import dispatch_once + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RECORD = ".loop/evidence/evidence-iter1.json" + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _task(task_id="T-1"): + return {"id": task_id, "title": task_id, "status": "pending", "criterion_ref": task_id, + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None} + + +def _dispatched(tmp_path, name, *, exit_code): + """One real dispatch whose verifier exits with ``exit_code``. + + A failing dispatch still writes and BINDS its evidence — that is the point: the + record is genuine provenance for a verification that did not pass. + """ + workspace = tmp_path / name + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": [_task()]}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": name}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + dispatch_once(workspace) + return workspace + + +def _rewrite_bundle(workspace, text): + """Replace the bundle bytes AND the record digest, so only the verdict is wrong. + + Without re-declaring the digest the record would fail hash verification and the + refusal would be right for the wrong reason. + """ + bundle = workspace / ".loop" / "artifacts" / "verify-iter1.json" + bundle.write_text(text, encoding="utf-8") + record_path = workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + record["sha256"] = hashlib.sha256(text.encode("utf-8")).hexdigest() + record_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return record_path + + +def _rebind(workspace, record_path): + """Re-bind the rewritten record at its new digest, alone, so the ambiguous-binding + refusal (the OTHER new check) cannot be what fires.""" + events = SQLiteEventStore(workspace / ".loop" / "events.db").read("run-1") + for suffix in ("", "-wal", "-shm"): + (workspace / ".loop" / ("events.db" + suffix)).unlink(missing_ok=True) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + digest = hashlib.sha256(record_path.read_bytes()).hexdigest() + bundle = workspace / ".loop" / "artifacts" / "verify-iter1.json" + bundle_digest = hashlib.sha256(bundle.read_bytes()).hexdigest() + for event in events: + hashes = event["artifact_hashes"] + if hashes: + hashes = [{"path": _RECORD, "sha256": digest}, + {"path": ".loop/artifacts/verify-iter1.json", "sha256": bundle_digest}] + store.append("run-1", event["type"], event["payload"], actor=event["actor"], + event_id=event["event_id"], causation_id=event["causation_id"], + correlation_id=event["correlation_id"], ts=event["ts"], + artifact_hashes=hashes or None) + + +def _terminal_file(workspace, entry): + """A hand-written strict terminal — the only way one citing a red verdict reaches + disk now that ``emit.terminate`` refuses to write it.""" + (workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "project": workspace.name, "state": "Succeeded", + "criteria_met": {"T-1": True}, "completion_policy": {"mode": VERIFIED_EVIDENCE_MODE}, + "evidence": [entry], "false_completion": False, + "terminated_at": "2026-07-25T00:00:00+00:00", "reason": "hand-written"}, + indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +# --- the reproduction --------------------------------------------------------- + + +def test_a_failing_verdict_cannot_back_succeeded_at_write_time(tmp_path): + workspace = _dispatched(tmp_path, "failing", exit_code=1) + bundle = json.loads((workspace / ".loop" / "artifacts" / "verify-iter1.json") + .read_text(encoding="utf-8")) + assert bundle["outcome"] == "FAIL" and bundle["passed"] is False # genuine red + with pytest.raises(emit.EmitError, match="verdict is not a pass"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_doctor_reports_a_strict_terminal_backed_by_a_failing_verdict(tmp_path): + workspace = _dispatched(tmp_path, "failing", exit_code=1) + _terminal_file(workspace, _RECORD) + report = doctor_report(workspace) + assert "unverified_evidence_terminal" in _codes(report) + assert any("verdict is not a pass" in issue["message"] for issue in report["issues"]) + + +def test_a_passing_dispatch_is_still_accepted(tmp_path): + """Positive control: the check narrows nothing that was honestly green.""" + workspace = _dispatched(tmp_path, "passing", exit_code=0) + path = emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} + + +# --- the green-marker rule, one definition ------------------------------------ + + +def test_metrics_and_the_completion_bar_share_one_green_marker_definition(): + assert metrics.verify_bundle_is_green is evidence.verify_bundle_is_green + + +@pytest.mark.parametrize("bundle,expected", [ + ({"outcome": "PASS"}, True), + ({"passed": True}, True), + ({"outcome": "pass"}, True), + ({"outcome": "FAIL", "passed": False}, False), + ({"score": 1.0}, False), # a score is not a verdict + ({"passed": 1}, False), # truthy is not True + ({}, False), +]) +def test_the_green_marker_rule(bundle, expected): + assert evidence.verify_bundle_is_green(bundle) is expected + + +def test_a_score_only_bundle_reads_red_at_the_strict_bar(tmp_path): + """The durable repo lesson, now load-bearing in two places at once: a bundle with a + perfect score and no verdict token is RED to metrics, and must be RED here too.""" + workspace = _dispatched(tmp_path, "scored", exit_code=0) + record_path = _rewrite_bundle(workspace, json.dumps({"score": 1.0}) + "\n") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="verdict is not a pass"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_a_self_contradicting_verdict_reads_green_pinned(tmp_path): + """The OR rule's cost, pinned rather than discovered later. + + ``verify_bundle_is_green`` is satisfied by EITHER token, so a bundle claiming + ``outcome: PASS`` while ``passed`` is false reads green and can back a strict + ``Succeeded``. That is the repo's canonical rule, shared object-for-object with + the FCR gate, so tightening it here alone would make one bundle green to the + completion bar and red to metrics — a worse defect than the one it fixes. + Changing it is a repo-wide decision about what a verdict means, not a patch. + """ + workspace = _dispatched(tmp_path, "contradicting", exit_code=0) + record_path = _rewrite_bundle( + workspace, json.dumps({"outcome": "PASS", "passed": False}) + "\n") + _rebind(workspace, record_path) + path = emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} # accepted + assert evidence.verify_bundle_is_green({"outcome": "PASS", "passed": False}) is True + + +# --- the non-verify-bundle kind, decided explicitly --------------------------- + + +def test_a_record_of_another_kind_cannot_back_succeeded(tmp_path): + """evidence@1's ``kind`` is an open vocabulary (log, diff, screenshot, report). + + A non-``verify-bundle`` record carries no verdict this layer can read, so it is + REFUSED rather than waved through: the strict mode's claim is that completion is + backed by a verification that passed, and an artifact with no verdict cannot make + that claim. + """ + workspace = _dispatched(tmp_path, "logkind", exit_code=0) + record_path = workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + record["kind"] = "log" + record_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="carries no verdict"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_an_unparseable_verdict_is_a_refusal_not_a_skip(tmp_path): + workspace = _dispatched(tmp_path, "garbage", exit_code=0) + record_path = _rewrite_bundle(workspace, "not json at all\n") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="not UTF-8 JSON"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_a_non_object_verdict_is_a_refusal(tmp_path): + workspace = _dispatched(tmp_path, "listy", exit_code=0) + record_path = _rewrite_bundle(workspace, "[]\n") + _rebind(workspace, record_path) + with pytest.raises(emit.EmitError, match="not a JSON object"): + emit.terminate(workspace, state="Succeeded", criteria_met={"T-1": True}, + evidence=[_RECORD], completion_policy=VERIFIED_EVIDENCE_MODE) + + +# --- the pure layers are unchanged, deliberately ------------------------------ + + +def test_the_reducer_still_admits_a_record_shaped_entry_without_reading_it(): + """Decision 14 stands: the reducer holds no filesystem, so it checks SHAPE only. + + Pinned so the verdict check is not later mistaken for something the pure fold + enforces — a fold that pretended to read a verdict would be the overclaim this + slice exists to prevent. + """ + projection = reduce_events([ + {"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 0, "event_id": "e0", + "type": "contract_opened", "actor": "op", "ts": "2026-07-25T00:00:00+00:00", + "causation_id": None, "correlation_id": None, "artifact_hashes": [], + "payload": {"workspace": "ws"}}, + {"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 1, "event_id": "e1", + "type": "terminal_written", "actor": "op", "ts": "2026-07-25T00:00:01+00:00", + "causation_id": None, "correlation_id": None, "artifact_hashes": [], + "payload": {"state": "Succeeded", "criteria_met": {"T-1": True}, "evidence": [_RECORD], + "false_completion": False, + "completion_policy": {"mode": VERIFIED_EVIDENCE_MODE}}}]) + assert projection["terminal"]["state"] == "Succeeded" diff --git a/scripts/test_completion_verified_evidence.py b/scripts/test_completion_verified_evidence.py new file mode 100644 index 0000000..f0e732b --- /dev/null +++ b/scripts/test_completion_verified_evidence.py @@ -0,0 +1,366 @@ +"""all_required_verified_evidence: the same bar at four layers, each as strong as it can be. + +Hash-verification ALONE is a forgeable bar: a worker with `.loop/` write access can +hand-write a bundle, its digest, a matching record and a correct policy_digest with +zero dispatch and zero events. The two layers that hold a workspace therefore also +require chain-boundness and live-goalpost agreement (plan decision 14); the two pure +layers keep only the structural half, because pretending otherwise is the failure +this mode exists to close. +""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop import emit, integrations +from loop.completion import (VERIFIED_EVIDENCE_MODE, evidence_entry_is_record_shaped, + normalize_completion_policy, policy_requires_verified_evidence) +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.reducer import EventReplayError, reduce_events +from loop.runner import dispatch_once +from loop.verifier import verification_policy_digest + +try: # the repo's fallback-mode convention + import jsonschema # noqa: F401 + _HAS_JSONSCHEMA = True +except ImportError: # pragma: no cover - exercised in the fallback leg + _HAS_JSONSCHEMA = False + +_MODES = [pytest.param("basic"), + pytest.param("strict", marks=pytest.mark.skipif(not _HAS_JSONSCHEMA, + reason="jsonschema not installed"))] + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + + +def _task(task_id, deps=(), status="pending", **extra): + return {"id": task_id, "title": task_id, "status": status, "criterion_ref": task_id, + "verify": _VERIFY, "depends_on": list(deps), "attempts": 0, "evidence": None, **extra} + + +def _ready(tmp_path, tasks): + """A contract projected to execute-task with the iteration counter still at 0. + + Same shape as scripts/test_evidence_binding_runner.py: the ramp records the FSM + walk, not work, so the first dispatch is iteration 1 and every artifact path + asserted below is literal. + """ + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": tasks}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + return workspace + + +@pytest.fixture +def ready_workspace(tmp_path): + return _ready(tmp_path, [_task("T-1")]) + + +@pytest.fixture +def dispatched_workspace(tmp_path): + """One real dispatch: `.loop/evidence/evidence-iter1.json` exists and IS chain-bound.""" + workspace = _ready(tmp_path, [_task("T-1")]) + dispatch_once(workspace) + return workspace + + +@pytest.fixture +def predone_workspace(tmp_path): + """Every task already declaratively done, so the auto-terminal fires with zero records.""" + return _ready(tmp_path, [_task("T-1", status="done", evidence="RUNLOG.md")]) + + +def test_the_new_mode_is_supported_and_normalizes(): + assert normalize_completion_policy(VERIFIED_EVIDENCE_MODE) == {"mode": VERIFIED_EVIDENCE_MODE} + assert policy_requires_verified_evidence({"mode": VERIFIED_EVIDENCE_MODE}) is True + + +def test_legacy_null_policy_still_normalizes_to_all_required(): + assert normalize_completion_policy(None) == {"mode": "all_required"} + assert policy_requires_verified_evidence(None) is False + + +def test_terminate_refuses_unverified_evidence_under_the_new_mode(dispatched_workspace): + with pytest.raises(emit.EmitError, match="verified evidence"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=["RUNLOG.md"], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_terminate_accepts_hash_verified_evidence_under_the_new_mode(dispatched_workspace): + path = emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": VERIFIED_EVIDENCE_MODE} + + +def test_terminate_under_the_default_mode_is_unchanged(dispatched_workspace): + path = emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=["RUNLOG.md"]) + assert json.loads(path.read_text(encoding="utf-8"))["completion_policy"] == { + "mode": "all_required"} + + +@pytest.mark.parametrize("mode", _MODES) +def test_doctor_reports_unverified_evidence_terminal(dispatched_workspace, mode): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + (dispatched_workspace / ".loop" / "artifacts" / "verify-iter1.json").write_text( + "{}", encoding="utf-8") + assert "unverified_evidence_terminal" in _codes(doctor_report(dispatched_workspace, mode=mode)) + + +# --- decision 14: hash-verified is NOT enough when the layer can reach further --- + +def _handwritten_record(workspace, *, name="evidence-iter9.json", task_id="T-1"): + """A self-consistent, never-dispatched record: real bundle, real digest, real policy.""" + bundle = workspace / ".loop" / "artifacts" / "verify-iter9.json" + bundle.parent.mkdir(parents=True, exist_ok=True) + text = '{"outcome": "PASS", "passed": true}' + bundle.write_text(text, encoding="utf-8") + tasks = json.loads((workspace / "TASKS.json").read_text(encoding="utf-8")) + entry = next(t for t in tasks["tasks"] if t["id"] == task_id) + record = { + "schema": "loop-engineer/evidence@1", "id": "hand:9:verify", "kind": "verify-bundle", + "uri": ".loop/artifacts/verify-iter9.json", + "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), + "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", + "produced_by": {"run_id": "run-1", "task_id": task_id, "attempt": 1, + "executor": "worker-a"}, + "verified_by": {"by": "ci", "at": "2026-07-25T00:00:00+00:00", + "command": entry["verify"], "code_digest": None, + "code_digest_basis": "path_lookup", + "policy_digest": verification_policy_digest(entry)}, + } + path = workspace / ".loop" / "evidence" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return f".loop/evidence/{name}" + + +def test_terminate_refuses_a_hash_verified_but_unbound_record_when_a_store_exists( + dispatched_workspace): + """The BLOCKER case: self-consistency is not dispatch. No event bound this record.""" + entry = _handwritten_record(dispatched_workspace) + with pytest.raises(emit.EmitError, match="not bound"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[entry], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_terminate_refuses_a_cited_record_whose_goalpost_moved_or_cannot_be_computed( + dispatched_workspace): + """Per-cited-record, deliberately stricter than doctor's latest-per-task rule. + + Both halves live in one test because the acceptance count is fixed at 17: a moved + goalpost and an UNCOMPUTABLE one are the same refusal, since a comparison that + cannot run has not passed (R007 — an errored check fails, it never skips). + """ + tasks_path = dispatched_workspace / "TASKS.json" + data = json.loads(tasks_path.read_text(encoding="utf-8")) + data["tasks"][0]["verify"] = "true" + tasks_path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(emit.EmitError, match="goalpost"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + + data["tasks"][0]["criterion_ref"] = float("nan") # json.loads accepts it; canonical JSON does not + tasks_path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(emit.EmitError, match="goalpost"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_doctor_reports_an_unbound_record_cited_by_a_strict_terminal(dispatched_workspace): + """Read-time twin of the write-time refusal — same predicate, one definition. + + The terminal file is hand-written, not emitted: emit.terminate REFUSES this exact + record (the test above), so the only way a strict terminal citing an unbound record + reaches disk is a worker writing it directly — which is the adversary being modelled. + """ + entry = _handwritten_record(dispatched_workspace) + (dispatched_workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "project": dispatched_workspace.name, + "state": "Succeeded", "criteria_met": {"C-1": True}, + "completion_policy": {"mode": VERIFIED_EVIDENCE_MODE}, "evidence": [entry], + "false_completion": False, "terminated_at": "2026-07-25T00:00:00+00:00", + "reason": "hand-written"}, indent=2, sort_keys=True) + "\n", encoding="utf-8") + assert "unverified_evidence_terminal" in _codes(doctor_report(dispatched_workspace)) + + +def test_the_write_layers_reject_an_entry_the_pure_layers_would_reject(tmp_path): + """The workspace-bearing bar must be a SUPERSET of the pure one. + + A store-less contract skips chain-boundness by design, so without an entry-shape + check a Succeeded terminal could cite a record living outside the workspace: written + happily by ``emit.terminate``, then refused by the reducer replaying that same + terminal. Layers that disagree about what Succeeded means are the defect. + """ + workspace = tmp_path / "storeless" + emit.open_contract(workspace) + outside = tmp_path / "elsewhere" / "rec.json" + outside.parent.mkdir(parents=True, exist_ok=True) + outside.write_text("{}", encoding="utf-8") + entry = "../elsewhere/rec.json" + assert evidence_entry_is_record_shaped(entry) is False + with pytest.raises(emit.EmitError, match="record path"): + emit.terminate(workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[entry], completion_policy=VERIFIED_EVIDENCE_MODE) + + +def _corrupt_store(workspace): + """Make the event store genuinely unreadable, at the bytes. + + Not a chmod: this suite runs as root in CI, where mode bits are advisory and the + 'unreadable' store would still open. Overwriting the header (and dropping the WAL + sidecars sqlite would otherwise recover from) is unreadable for everyone. + """ + loop_dir = workspace / ".loop" + for sidecar in ("events.db-wal", "events.db-shm"): + (loop_dir / sidecar).unlink(missing_ok=True) + (loop_dir / "events.db").write_bytes(b"this is not a sqlite database" * 64) + + +def test_terminate_refuses_the_new_mode_when_the_event_store_cannot_be_read(dispatched_workspace): + """Fail-closed, not fail-open: a chain nobody can read binds nothing (R007).""" + _corrupt_store(dispatched_workspace) + with pytest.raises(emit.EmitError, match="event store could not be read"): + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + + +def test_doctor_reports_a_strict_terminal_whose_event_store_cannot_be_read(dispatched_workspace): + """The read-time twin degrades to a FINDING rather than an exception: doctor's job is + to report every issue it can see, so an unreadable store must not abort the sweep.""" + emit.terminate(dispatched_workspace, state="Succeeded", criteria_met={"C-1": True}, + evidence=[".loop/evidence/evidence-iter1.json"], + completion_policy=VERIFIED_EVIDENCE_MODE) + _corrupt_store(dispatched_workspace) + report = doctor_report(dispatched_workspace) # must not raise + assert "unverified_evidence_terminal" in _codes(report) + # Pinned to the store-read branch: the cited record itself is still perfectly valid, + # so any OTHER path to this code would mean the finding is right for the wrong reason. + assert any("event store could not be read" in issue["message"] + for issue in report["issues"] + if issue["code"] == "unverified_evidence_terminal") + + +def _terminal_event(evidence, mode): + return [{"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 0, "event_id": "e0", + "type": "contract_opened", "actor": "op", "ts": "2026-07-25T00:00:00+00:00", + "causation_id": None, "correlation_id": None, "payload": {"workspace": "ws"}, + "artifact_hashes": []}, + {"schema": "loop-engineer/event@1", "run_id": "r", "sequence": 1, "event_id": "e1", + "type": "terminal_written", "actor": "op", "ts": "2026-07-25T00:00:01+00:00", + "causation_id": None, "correlation_id": None, "artifact_hashes": [], + "payload": {"state": "Succeeded", "criteria_met": {"C-1": True}, + "evidence": evidence, "false_completion": False, + "completion_policy": {"mode": mode}}}] + + +def test_the_reducer_refuses_a_non_record_evidence_entry_under_the_new_mode(): + with pytest.raises(EventReplayError, match="verified evidence"): + reduce_events(_terminal_event(["RUNLOG.md"], VERIFIED_EVIDENCE_MODE)) + + +def test_the_reducer_accepts_record_shaped_evidence_under_the_new_mode(): + projection = reduce_events( + _terminal_event([".loop/evidence/evidence-iter1.json"], VERIFIED_EVIDENCE_MODE)) + assert projection["terminal"]["state"] == "Succeeded" + + +def test_integrations_projection_refuses_non_record_evidence_under_the_new_mode(): + outcome = integrations.EngineOutcome(reached_end=True, artifacts=("RUNLOG.md",)) + body = integrations.to_terminal_state( + outcome, {"verdict": "Succeeded", "passed_visible": True, "passed_holdout": True, + "false_completion": False, + "visible": [{"id": "C-1", "passed": True}], + "holdout": [{"id": "H-1", "passed": True}]}, + # A clean anticheat result needs `downgrade_to` present, or the projection fails + # closed on the anticheat input and never reaches the evidence bar under test. + {"findings": [], "clean": True, "downgrade_to": None}, {"C-1": True}, + completion_policy=VERIFIED_EVIDENCE_MODE) + assert body["state"] == "FailedUnverifiable" and "verified evidence" in body["reason"] + + +def test_runner_auto_terminal_adopts_the_verified_mode_only_when_the_records_pass_the_bar( + tmp_path, ready_workspace): + """Existence is not satisfaction, and the check runs BEFORE the event is appended. + + Both halves live in one test because the acceptance count is fixed: they are the two + sides of a single decision — the runner evaluates the shared predicate against every + candidate record, then chooses the mode. Evaluating it afterwards (what `emit` + already did) would leave `terminal_written` durable and then raise, which is exactly + the committed-then-refused seam this slice removes. + """ + dispatch_once(ready_workspace) + dispatch_once(ready_workspace) + terminal = json.loads((ready_workspace / ".loop" / "terminal_state.json") + .read_text(encoding="utf-8")) + assert terminal["completion_policy"] == {"mode": VERIFIED_EVIDENCE_MODE} + assert terminal["evidence"] == [".loop/evidence/evidence-iter1.json"] + + moved = _ready(tmp_path / "moved", [_task("T-1")]) + dispatch_once(moved) + tasks_path = moved / "TASKS.json" # move the goalpost the record cites + data = json.loads(tasks_path.read_text(encoding="utf-8")) + data["tasks"][0]["verify"] = "true" + tasks_path.write_text(json.dumps(data), encoding="utf-8") + result = dispatch_once(moved) # must not raise, and must not lie + assert result["action"] == "terminal_written" + written = json.loads((moved / ".loop" / "terminal_state.json").read_text(encoding="utf-8")) + assert written["completion_policy"] == {"mode": "all_required"} + assert written["evidence"] == ["RUNLOG.md"] + assert ".loop/evidence/evidence-iter1.json" in written["reason"] + assert "goalpost" in written["reason"] + # The durable event says the same thing the file does: one decision, taken once. + event = SQLiteEventStore(moved / ".loop" / "events.db").read("run-1")[-1] + assert event["type"] == "terminal_written" + assert event["payload"]["completion_policy"] == {"mode": "all_required"} + assert event["payload"]["reason"] == written["reason"] + + +def test_runner_auto_terminal_keeps_all_required_when_a_task_has_no_record(predone_workspace): + dispatch_once(predone_workspace) + terminal = json.loads((predone_workspace / ".loop" / "terminal_state.json") + .read_text(encoding="utf-8")) + assert terminal["completion_policy"] == {"mode": "all_required"} + assert terminal["evidence"] == ["RUNLOG.md"] + assert "no evidence record" in terminal["reason"] + + +@pytest.mark.parametrize("mode", _MODES) +def test_terminal_schema_accepts_both_modes(ready_workspace, mode): + """terminal@1 admits the new mode in BOTH validation modes, end to end. + + Driven through the runner's own auto-terminal rather than a bare emit.terminate: + a hand-terminated dispatched workspace is doctor-dirty for unrelated reasons (a + terminal file with no terminal_written event is a desynced terminal window), so + `ok is True` there would prove nothing about the schema. + """ + dispatch_once(ready_workspace) + dispatch_once(ready_workspace) + report = doctor_report(ready_workspace, mode=mode) + assert report["ok"] is True, report["issues"] diff --git a/scripts/test_conformance.py b/scripts/test_conformance.py index d26f389..90a7380 100644 --- a/scripts/test_conformance.py +++ b/scripts/test_conformance.py @@ -436,3 +436,123 @@ def test_documented_policy_digest_vector_matches_the_implementation(): assert verification_policy_digest(_POLICY_VECTOR_TASK) == _POLICY_VECTOR_DIGEST doc = (Path(__file__).resolve().parent.parent / "reference" / "repo-os-contract.md").read_text(encoding="utf-8") assert _POLICY_VECTOR_DIGEST in doc and _POLICY_VECTOR_CANONICAL in doc + + +# --------------------------------------------------------------------------- # +# Evidence-wiring doc pins — the shipped surfaces and the normative doc cannot +# drift apart. Every assertion below was proven to FAIL against the tree as it +# stood before the documentation commit; a pin that passes both before and +# after documents nothing. +# --------------------------------------------------------------------------- # + +from loop import emit # noqa: E402 +from loop.verifier import injected_verifier_identity # noqa: E402 + +_CONTRACT_DOC = ROOT / "reference" / "repo-os-contract.md" + +# The retired sentence was written three different ways on the three surfaces +# ("does NOT yet hash-verify", "does not yet hash-verify", "does not +# hash-verify"), so the ban is case-insensitive and the "yet" is optional. A +# case-sensitive substring ban passes against the uppercase one and proves +# nothing about it. +_RETIRED_HASH_VERIFY_CLAIM = re.compile(r"does\s+not\s+(?:yet\s+)?hash-verify", re.IGNORECASE) + + +def _binding_fixture(tmp_path: Path) -> Path: + """A minimal in-flight contract that ``build_verify_evidence`` can render + against — it needs only a readable ``.loop/state.json``.""" + return _scaffold_inflight(tmp_path / "binding") + + +def _bound_evidence_section() -> str: + """§17's ``### Bound evidence`` subsection, heading to next heading.""" + text = _CONTRACT_DOC.read_text(encoding="utf-8") + assert "### Bound evidence" in text, "§17 does not document the bound evidence set" + start = text.index("### Bound evidence") + end = text.find("\n### ", start + 1) + return text[start:] if end == -1 else text[start:end] + + +def test_documented_artifact_binding_vector_matches_the_writer(tmp_path): + """The three bound paths and their order are normative, not incidental.""" + workspace = _binding_fixture(tmp_path) + built = emit.build_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_POLICY_VECTOR_TASK, passed=True, + code_identity=injected_verifier_identity()) + # Order is the WRITER'S CONVENTION, not a correctness property — the doctor + # walk builds a path->digest map and `event_hash` recomputes over the stored + # list exactly as written, so any order verifies. It is pinned to the SHIPPED + # order (bundle -> record -> object) purely so it cannot drift silently away + # from the ordered list §17 publishes. + assert [entry["path"] for entry in built.artifact_hashes] == [ + ".loop/artifacts/verify-iter1.json", + ".loop/evidence/evidence-iter1.json", + f".loop/artifacts/objects/{built.sha256[:2]}/{built.sha256}", + ] + # And the doc's numbered list must enumerate the same three in the same order. + section = _bound_evidence_section() + markers = (".loop/artifacts/verify-iter", ".loop/evidence/evidence-iter", + ".loop/artifacts/objects/") + for marker in markers: + assert marker in section, f"§17 bound-evidence set omits {marker}" + positions = [section.index(marker) for marker in markers] + assert positions == sorted(positions), ( + f"§17 lists the bound set out of writer order: {positions}") + + +def _new_issue_codes_table() -> str: + """§22's ``**New issue codes.**`` table, heading row to the end of the table.""" + text = _CONTRACT_DOC.read_text(encoding="utf-8") + marker = "**New issue codes.**" + assert marker in text, "§22 does not publish a new-issue-codes table" + start = text.index(marker) + end = text.find("\n\n", text.index("|---|---|", start)) + return text[start:] if end == -1 else text[start:end] + + +def test_every_new_doctor_code_is_documented_in_section_22(): + """Scoped to the §22 TABLE, not the whole file. + + Grepping the whole document could not fail: every one of these codes also appears + in §17's tier list, so deleting all four §22 rows left the pin green — the pin a PR + body would cite as proof that the codes are documented. + """ + table = _new_issue_codes_table() + for code in ("evidence_chain_mismatch", "missing_bound_evidence", + "policy_digest_mismatch", "unverified_evidence_terminal", + "bound_evidence_escape"): + assert f"`{code}`" in table, f"§22's new-issue-codes table does not document {code}" + + +def test_no_shipped_surface_still_claims_evidence_is_unverified(): + for path in (ROOT / "loop" / "evidence.py", ROOT / "schemas" / "evidence.schema.json", + _CONTRACT_DOC): + text = path.read_text(encoding="utf-8") + assert not _RETIRED_HASH_VERIFY_CLAIM.search(text), ( + f"{path.name} still disclaims hash verification") + # Whitespace-normalised: the retired tier bullet wrapped between the "**" + # and `policy_digest`, so a raw substring ban never matched it either. + normalised = " ".join(text.split()) + assert "not checked by any shipped surface:** `policy_digest`" not in normalised, ( + f"{path.name} still files policy_digest under the unchecked tier") + # decision 14: the strict mode's two residuals must be named where the mode is + # documented, not only in this plan. A capability paragraph without its limits is + # the overclaim this slice exists to prevent. + contract = " ".join(_CONTRACT_DOC.read_text(encoding="utf-8").split()) + assert "no event store" in contract and "--expect-chain-head" in contract + # §17 closes with the trust-domain sentence rather than the plan's + # "detectable against an anchor" wording (that phrasing ships in the README). + # Pinned against the sentence the contract actually carries — absent before + # this release, so the assertion discriminates. + assert "trust domain can anchor" in contract + # Deliberately NOT asserting the absence of "tamper-proof": §17 names it as the + # forbidden claim, so a substring ban would fire on the honesty sentence itself. + + +def test_section_16_event_type_list_matches_the_code(): + from loop.events import EVENT_TYPES + doc = _CONTRACT_DOC.read_text(encoding="utf-8") + # §16 renders the nine members inside one wrapped backtick span, so a bare + # name is as good as a backticked one here. + assert all(f"`{name}`" in doc or name in doc for name in EVENT_TYPES) + assert "one-to-one with `loop.emit`'s four writer operations" not in doc diff --git a/scripts/test_doctor_evidence.py b/scripts/test_doctor_evidence.py index 0475e50..be04abe 100644 --- a/scripts/test_doctor_evidence.py +++ b/scripts/test_doctor_evidence.py @@ -14,10 +14,17 @@ def _codes(report): return {issue["code"] for issue in report["issues"]} +_BUNDLE_TEXT = json.dumps({"outcome": "PASS", "passed": True}) +_BUNDLE_SHA = hashlib.sha256(_BUNDLE_TEXT.encode("utf-8")).hexdigest() +# A legacy bundle name: doctor now hash-verifies a record's uri, so the file must +# exist, and a runner-shaped verify-iter.json would need its own record. +_RECORD_BUNDLE_NAME = "verify-T1.json" + + def _record(executor="worker-a", by="ci", **overrides): record = { "schema": "loop-engineer/evidence@1", "id": "e1", "kind": "verify-bundle", - "uri": ".loop/artifacts/verify-iter5.json", "sha256": "c" * 64, + "uri": f".loop/artifacts/{_RECORD_BUNDLE_NAME}", "sha256": _BUNDLE_SHA, "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", "produced_by": {"run_id": "run-1", "task_id": "T-1", "attempt": 1, "executor": executor}, "verified_by": {"by": by, "at": "2026-07-25T00:00:00+00:00", @@ -32,17 +39,20 @@ def _record(executor="worker-a", by="ci", **overrides): def _ws(tmp_path, records=(), bundles=(), name="workspace"): target = tmp_path / name scaffold(target) + artifacts = target / ".loop" / "artifacts" if records: directory = target / ".loop" / "evidence" directory.mkdir(parents=True, exist_ok=True) for index, record in enumerate(records): text = record if isinstance(record, str) else json.dumps(record) (directory / f"evidence-iter{index}.json").write_text(text, encoding="utf-8") + # Every _record() names this uri, so it must exist with exactly the bytes + # the record commits to. + artifacts.mkdir(parents=True, exist_ok=True) + (artifacts / _RECORD_BUNDLE_NAME).write_text(_BUNDLE_TEXT, encoding="utf-8") for bundle_name in bundles: - directory = target / ".loop" / "artifacts" - directory.mkdir(parents=True, exist_ok=True) - (directory / bundle_name).write_text( - json.dumps({"outcome": "PASS", "passed": True}), encoding="utf-8") + artifacts.mkdir(parents=True, exist_ok=True) + (artifacts / bundle_name).write_text(_BUNDLE_TEXT, encoding="utf-8") return target diff --git a/scripts/test_doctor_evidence_binding.py b/scripts/test_doctor_evidence_binding.py new file mode 100644 index 0000000..ffe03d9 --- /dev/null +++ b/scripts/test_doctor_evidence_binding.py @@ -0,0 +1,189 @@ +"""Doctor's chain-binding walk: bound artifacts must still be the bytes that were bound.""" +from __future__ import annotations + +import json + +import pytest + +from loop import emit +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.runner import dispatch_once +from loop.scaffold import scaffold + +try: # the repo's fallback-mode convention + import jsonschema # noqa: F401 + _HAS_JSONSCHEMA = True +except ImportError: # pragma: no cover - exercised in the fallback leg + _HAS_JSONSCHEMA = False + +_MODES = [pytest.param("basic"), + pytest.param("strict", marks=pytest.mark.skipif(not _HAS_JSONSCHEMA, + reason="jsonschema not installed"))] + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_RUN_ID = "run-1" + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _tree(workspace): + return sorted(str(p.relative_to(workspace)) for p in workspace.rglob("*") if p.is_file()) + + +def _store_path(workspace): + return workspace / ".loop" / "events.db" + + +def _ready(tmp_path): + """A contract projected to execute-task with the iteration counter still at 0. + + The ramp records the FSM walk to execute-task, not work: intake is reachable + only by iteration_appended, so the ramp events exist, but they carry + iteration_id 0 and no task verdict. The first dispatch is therefore iteration + 1 and every artifact path asserted below is literal. + """ + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [{"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "T-1", + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None}], + }), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(_store_path(workspace)) + store.append(_RUN_ID, "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append(_RUN_ID, "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + # The ramp is written straight to the store, so state.json still says intake. + # Doctor reconciles the two, so the fixture must land where the ramp landed. + state_path = workspace / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["state"] = _RAMP[-1] + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + return workspace + + +def _rebuild_unbound(workspace): + """Replay the stream into a fresh store with nothing bound. + + The append-only triggers make retroactive unbinding impossible, so the only + way a pre-binding contract can exist is the way it was originally written: + every event re-appended verbatim except its artifact_hashes. + """ + events = SQLiteEventStore(_store_path(workspace)).read(_RUN_ID) + for suffix in ("", "-wal", "-shm"): + _store_path(workspace).with_name("events.db" + suffix).unlink(missing_ok=True) + store = SQLiteEventStore(_store_path(workspace)) + for event in events: + store.append(_RUN_ID, event["type"], event["payload"], actor=event["actor"], + event_id=event["event_id"], causation_id=event["causation_id"], + correlation_id=event["correlation_id"], ts=event["ts"], + artifact_hashes=None) + + +@pytest.fixture +def scaffolded_workspace(tmp_path): + target = tmp_path / "scaffolded" + scaffold(target) + return target + + +@pytest.fixture +def ready_workspace(tmp_path): + return _ready(tmp_path) + + +@pytest.fixture +def legacy_workspace(tmp_path): + workspace = _ready(tmp_path) + dispatch_once(workspace) + _rebuild_unbound(workspace) + return workspace + + +@pytest.fixture +def corrupt_store_workspace(tmp_path): + target = tmp_path / "corrupt" + scaffold(target) + _store_path(target).write_text("not sqlite", encoding="utf-8") + return target + + +def test_absent_store_adds_no_binding_issue(scaffolded_workspace): + assert doctor_report(scaffolded_workspace)["event_store"] == {"present": False} + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_clean_runner_dispatch_is_doctor_clean(ready_workspace, mode): + dispatch_once(ready_workspace) + report = doctor_report(ready_workspace, mode=mode) + assert report["ok"] is True and report["issues"] == [] + + +def test_rewriting_a_bound_bundle_is_reported(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").write_text( + json.dumps({"outcome": "PASS", "passed": True}), encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(ready_workspace)) + + +def test_rewriting_a_bound_record_is_reported(ready_workspace): + dispatch_once(ready_workspace) + path = ready_workspace / ".loop" / "evidence" / "evidence-iter1.json" + record = json.loads(path.read_text(encoding="utf-8")) + record["verified_by"]["by"] = "somebody-else" + path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + assert "evidence_chain_mismatch" in _codes(doctor_report(ready_workspace)) + + +def test_deleting_a_bound_pair_is_reported(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").unlink() + (ready_workspace / ".loop" / "evidence" / "evidence-iter1.json").unlink() + assert "missing_bound_evidence" in _codes(doctor_report(ready_workspace)) + + +def test_the_object_survives_and_is_named_as_the_recovery_source(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").write_text("{}", encoding="utf-8") + messages = " ".join(issue["message"] for issue in doctor_report(ready_workspace)["issues"]) + assert "objects/" in messages + assert any(p.startswith(".loop/artifacts/objects/") + for p in _tree(ready_workspace)) + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_legacy_event_without_artifact_hashes_is_not_a_finding(legacy_workspace, mode): + report = doctor_report(legacy_workspace, mode=mode) + assert "missing_bound_evidence" not in _codes(report) + assert "evidence_chain_mismatch" not in _codes(report) + + +def test_binding_check_writes_nothing(ready_workspace): + dispatch_once(ready_workspace) + before = _tree(ready_workspace) + doctor_report(ready_workspace) + assert _tree(ready_workspace) == before + + +def test_binding_issues_never_add_a_doctor_key(ready_workspace): + dispatch_once(ready_workspace) + (ready_workspace / ".loop" / "artifacts" / "verify-iter1.json").unlink() + report = doctor_report(ready_workspace) + assert set(report["event_store"]) == { + "present", "readable", "run_id", "event_count", "state_json_agrees", + "deterministic", "legal_sequence", "chain"} + + +def test_a_store_that_cannot_be_read_still_reports_its_own_error_code(corrupt_store_workspace): + report = doctor_report(corrupt_store_workspace) + assert report["event_store"]["readable"] is False + assert "missing_bound_evidence" not in _codes(report) diff --git a/scripts/test_doctor_evidence_verification.py b/scripts/test_doctor_evidence_verification.py new file mode 100644 index 0000000..ebe992f --- /dev/null +++ b/scripts/test_doctor_evidence_verification.py @@ -0,0 +1,211 @@ +"""Doctor hash-verifies discovered records and compares their policy digest to TASKS.json.""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop.contract import doctor_report +from loop.scaffold import scaffold +from loop.verifier import verification_policy_digest + +try: # the repo's fallback-mode convention + import jsonschema # noqa: F401 + _HAS_JSONSCHEMA = True +except ImportError: # pragma: no cover - exercised in the fallback leg + _HAS_JSONSCHEMA = False + +_MODES = [pytest.param("basic"), + pytest.param("strict", marks=pytest.mark.skipif(not _HAS_JSONSCHEMA, + reason="jsonschema not installed"))] + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +_TASK = {"id": "T-1", "title": "t", "status": "pending", "criterion_ref": "C-1", + "verify": "./scripts/verify-fast.sh", "depends_on": [], "attempts": 0, "evidence": None} + + +def _ws(tmp_path, *, records=(), named_records=(), + bundle_text='{"outcome": "PASS", "passed": true}', tasks=(_TASK,)): + target = tmp_path / "workspace" + scaffold(target) + (target / "TASKS.json").write_text(json.dumps( + {"schema": "loop-engineer/tasks@1", "project": "p", "tasks": list(tasks)}), encoding="utf-8") + # scaffold ships scripts/verify-fast without an extension; _TASK declares the + # .sh form, and a path-shaped task verify must resolve under the workspace. + (target / "scripts" / "verify-fast.sh").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + (target / ".loop" / "artifacts").mkdir(parents=True, exist_ok=True) + (target / ".loop" / "artifacts" / "verify-iter1.json").write_text(bundle_text, encoding="utf-8") + directory = target / ".loop" / "evidence" + directory.mkdir(parents=True, exist_ok=True) + for iteration_id, record in records: + (directory / f"evidence-iter{iteration_id}.json").write_text( + json.dumps(record), encoding="utf-8") + for name, record in named_records: + (directory / name).write_text(json.dumps(record), encoding="utf-8") + return target + + +def _record(*, iteration_id=1, policy_digest=None, uri=".loop/artifacts/verify-iter1.json", + sha256=None, task_id="T-1"): + text = '{"outcome": "PASS", "passed": true}' + return { + "schema": "loop-engineer/evidence@1", "id": f"run-1:{iteration_id}:verify", + "kind": "verify-bundle", "uri": uri, + "sha256": sha256 or hashlib.sha256(text.encode("utf-8")).hexdigest(), + "media_type": "application/json", "created_at": "2026-07-25T00:00:00+00:00", + "produced_by": {"run_id": "run-1", "task_id": task_id, "attempt": 1, + "executor": "worker-a"}, + "verified_by": {"by": "ci", "at": "2026-07-25T00:00:00+00:00", + "command": "./scripts/verify-fast.sh", "code_digest": None, + "code_digest_basis": "path_lookup", + "policy_digest": policy_digest or verification_policy_digest(_TASK)}, + } + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_hash_verified_record_is_clean(tmp_path, mode): + target = _ws(tmp_path, records=[(1, _record())]) + assert doctor_report(target, mode=mode)["ok"] is True + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_swapped_bundle_fails_doctor_with_hash_mismatch(tmp_path, mode): + target = _ws(tmp_path, records=[(1, _record())]) + (target / ".loop" / "artifacts" / "verify-iter1.json").write_text("{}", encoding="utf-8") + assert "hash_mismatch" in _codes(doctor_report(target, mode=mode)) + + +def test_a_record_whose_uri_is_absent_reports_missing_evidence_path(tmp_path): + target = _ws(tmp_path, records=[(1, _record(uri=".loop/artifacts/gone.json"))]) + assert "missing_evidence_path" in _codes(doctor_report(target)) + + +def test_a_record_whose_uri_escapes_the_workspace_reports_workspace_escape(tmp_path): + target = _ws(tmp_path, records=[(1, _record(uri="../escape.json"))]) + assert _codes(doctor_report(target)) & {"workspace_escape", "missing_evidence_path"} + + +@pytest.mark.parametrize("mode", _MODES) +def test_policy_digest_mismatch_against_the_live_task(tmp_path, mode): + moved = dict(_TASK, verify="true") + target = _ws(tmp_path, records=[(1, _record())], tasks=(moved,)) + assert "policy_digest_mismatch" in _codes(doctor_report(target, mode=mode)) + + +def test_policy_digest_agreement_is_silent(tmp_path): + target = _ws(tmp_path, records=[(1, _record())]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_only_the_latest_record_per_task_is_compared(tmp_path): + stale = _record(iteration_id=1, policy_digest="d" * 64) + current = _record(iteration_id=2) + target = _ws(tmp_path, records=[(1, stale), (2, current)]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +# --- unnumbered records are ranked, never excluded ---------------------------- +# +# The sweep used to `continue` on any record outside `evidence-iter.json`, so a +# task whose ONLY record carried no iteration id was never compared: move its +# goalpost and doctor stayed ok with zero issues, while the identical record renamed +# `evidence-iter9.json` fired. Being unnumbered was a way out of the comparison, and +# §22 meanwhile claimed "the latest evidence record for a task" is compared. Latest +# now RANKS the unnumbered below every numbered record (ties by filename) rather +# than dropping them, so the only record for a task is always compared. + + +def test_a_tasks_only_record_is_compared_even_when_it_is_unnumbered(tmp_path): + moved = dict(_TASK, verify="true") + target = _ws(tmp_path, named_records=[("evidence-hand.json", _record())], tasks=(moved,)) + assert "policy_digest_mismatch" in _codes(doctor_report(target)) + + +def test_an_unnumbered_record_is_silent_when_its_goalpost_agrees(tmp_path): + """Positive control: ranking them in did not make every hand-written record a finding. + + Scoped to the goalpost code — the fixture's numbered BUNDLE legitimately reports + ``missing_evidence_record`` (no ``evidence-iter1.json`` describes it), which is a + different check and not what this test is about. + """ + target = _ws(tmp_path, named_records=[("evidence-hand.json", _record())]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_a_numbered_record_outranks_an_unnumbered_one_for_the_same_task(tmp_path): + """An unnumbered record carries no position in the run's order, so it can never be + the latest while a numbered record exists — the stale digest below is not compared.""" + target = _ws(tmp_path, + records=[(2, _record(iteration_id=2))], + named_records=[("evidence-hand.json", _record(policy_digest="d" * 64))]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_unnumbered_records_tie_break_by_filename(tmp_path): + """With no ordering to appeal to, filename order is the deterministic tie-break — + the same rule the numbered path already used for equal iteration ids.""" + target = _ws(tmp_path, named_records=[ + ("evidence-a.json", _record(policy_digest="d" * 64)), + ("evidence-b.json", _record()), + ]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + later_is_stale = _ws(tmp_path / "second", named_records=[ + ("evidence-a.json", _record()), + ("evidence-b.json", _record(policy_digest="d" * 64)), + ]) + assert "policy_digest_mismatch" in _codes(doctor_report(later_is_stale)) + + +def test_an_unnumbered_record_naming_an_unknown_task_is_still_not_compared(tmp_path): + """Decision 5 is untouched: a renamed or removed task is a replan, not a forgery.""" + target = _ws(tmp_path, named_records=[ + ("evidence-hand.json", _record(task_id="T-404", policy_digest="d" * 64))]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_a_record_naming_an_unknown_task_is_not_compared(tmp_path): + target = _ws(tmp_path, records=[(1, _record(task_id="T-404", policy_digest="d" * 64))]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def test_a_record_without_a_policy_digest_is_not_compared(tmp_path): + record = _record() + record["verified_by"]["policy_digest"] = None + target = _ws(tmp_path, records=[(1, record)]) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +def _nan_task(): + """A live entry json.loads accepts but verification_policy_digest cannot canonicalize.""" + return dict(_TASK, criterion_ref=float("nan")) + + +@pytest.mark.parametrize("mode", _MODES) +def test_a_non_canonicalizable_live_task_fails_closed_in_both_modes(tmp_path, mode): + """An unrunnable comparison is not an agreement. + + jsonschema mode independently reports the malformed entry as a schema_violation, + but structural-fallback has no tasks@1 type-check — skipping here left a stale + goalpost reported by nothing, and doctor answered ok. + """ + target = _ws(tmp_path, records=[(1, _record(policy_digest="d" * 64))], + tasks=(_nan_task(),)) + report = doctor_report(target, mode=mode) + assert "policy_digest_mismatch" in _codes(report) + assert report["ok"] is False + + +def test_a_non_canonicalizable_task_no_record_names_is_not_a_goalpost_finding(tmp_path): + target = _ws(tmp_path, records=[(1, _record(task_id="T-404"))], tasks=(_nan_task(),)) + assert "policy_digest_mismatch" not in _codes(doctor_report(target)) + + +@pytest.mark.parametrize("mode", _MODES) +def test_evidence_schema_id_still_joins_schemas_checked(tmp_path, mode): + target = _ws(tmp_path, records=[(1, _record())]) + assert "loop-engineer/evidence@1" in doctor_report(target, mode=mode)["schemas_checked"] diff --git a/scripts/test_doctor_store_read_guard.py b/scripts/test_doctor_store_read_guard.py new file mode 100644 index 0000000..e48c325 --- /dev/null +++ b/scripts/test_doctor_store_read_guard.py @@ -0,0 +1,55 @@ +"""Doctor's THIRD store read is guarded exactly like its two siblings (R007). + +``event_consistency_issues`` reads the store three times: ``status_report``, +``replay_report``, and the chain-binding walk. The first two sat inside an +``except RuntimeStoreError`` that turns an unreadable store into a typed finding; the +walk sat OUTSIDE it, so a store that became unreadable between reads escaped +``doctor_report`` as an untyped traceback — an errored check must fail, never explode. +""" +from __future__ import annotations + +import json + +from loop import emit +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.runner import dispatch_once +from loop.runtime import RuntimeStoreError + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + + +def _dispatched(tmp_path): + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text(json.dumps({ + "schema": "loop-engineer/tasks@1", + "tasks": [{"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "T-1", + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None}], + }), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + dispatch_once(workspace) + return workspace + + +def test_a_store_that_dies_between_reads_is_a_typed_finding_not_a_traceback( + tmp_path, monkeypatch): + workspace = _dispatched(tmp_path) + + def boom(*args, **kwargs): + raise RuntimeStoreError("corrupt_store", "the store died between reads") + + monkeypatch.setattr("loop.runtime._bound_evidence_issues", boom) + report = doctor_report(workspace) # must not raise + assert "corrupt_store" in {issue["code"] for issue in report["issues"]} + assert report["event_store"] == {"present": True, "readable": False, + "error_code": "corrupt_store"} diff --git a/scripts/test_evidence_binding_runner.py b/scripts/test_evidence_binding_runner.py new file mode 100644 index 0000000..34285e9 --- /dev/null +++ b/scripts/test_evidence_binding_runner.py @@ -0,0 +1,139 @@ +"""dispatch_once binds its evidence digests into the iteration event before writing them.""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop import emit +from loop.chain import compute_event_hash +from loop.events import SQLiteEventStore +from loop.runner import RunnerError, VerifyOutcome, dispatch_once + +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") + + +def _task(i, deps=(), status="pending"): + return {"id": i, "title": i, "status": status, "criterion_ref": i, "verify": _VERIFY, + "depends_on": list(deps), "attempts": 0, "evidence": None} + + +def _ready(tmp_path, tasks): + """A contract projected to execute-task with the iteration counter still at 0. + + The ramp records the FSM walk to execute-task, not work: intake is reachable + only by iteration_appended, so the ramp events exist, but they carry + iteration_id 0 and no task verdict. The first dispatch is therefore iteration + 1 and every artifact path asserted below is literal. + """ + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": tasks}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + store = SQLiteEventStore(workspace / ".loop" / "events.db") + store.append("run-1", "contract_opened", {"workspace": "workspace"}, actor="test") + for state in _RAMP: + store.append("run-1", "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + return workspace + + +@pytest.fixture +def ready_workspace(tmp_path): + return _ready(tmp_path, [_task("T-1")]) + + +@pytest.fixture +def two_task_workspace(tmp_path): + return _ready(tmp_path, [_task("T-1"), _task("T-2", ("T-1",))]) + + +def _events(workspace, run_id="run-1"): + return SQLiteEventStore(workspace / ".loop" / "events.db").read(run_id) + + +def _iteration_events(workspace): + """Iterations carrying a task verdict — exactly the ones a dispatch appends.""" + return [e for e in _events(workspace) if e["type"] == "iteration_appended" + and e["payload"]["outcome"] in ("task_passed", "task_failed")] + + +def test_dispatch_binds_the_evidence_digests_into_the_iteration_event(ready_workspace): + dispatch_once(ready_workspace) + hashes = _iteration_events(ready_workspace)[-1]["artifact_hashes"] + paths = {entry["path"] for entry in hashes} + assert len(hashes) == 3 + assert ".loop/artifacts/verify-iter1.json" in paths + assert ".loop/evidence/evidence-iter1.json" in paths + + +def test_bound_digests_match_the_files_on_disk(ready_workspace): + dispatch_once(ready_workspace) + for entry in _iteration_events(ready_workspace)[-1]["artifact_hashes"]: + blob = (ready_workspace / entry["path"]).read_bytes() + assert hashlib.sha256(blob).hexdigest() == entry["sha256"] + + +def test_the_bound_event_hash_covers_the_artifact_hashes(ready_workspace): + dispatch_once(ready_workspace) + event = _iteration_events(ready_workspace)[-1] + assert compute_event_hash(event) == event["event_hash"] + tampered = {**event, "artifact_hashes": []} + assert compute_event_hash(tampered) != event["event_hash"] + + +def test_a_build_failure_commits_no_event(ready_workspace): + tasks_path = ready_workspace / "TASKS.json" + data = json.loads(tasks_path.read_text(encoding="utf-8")) + data["tasks"][0]["criterion_ref"] = float("nan") + tasks_path.write_text(json.dumps(data), encoding="utf-8") + before = len(_events(ready_workspace)) + with pytest.raises(RunnerError): + dispatch_once(ready_workspace) + assert len(_events(ready_workspace)) == before + + +def test_an_evidence_write_failure_after_the_commit_is_still_loud(ready_workspace, monkeypatch): + def boom(*args, **kwargs): + raise OSError("disk full") + monkeypatch.setattr(emit, "write_verify_evidence", boom) + with pytest.raises(RunnerError, match="committed to the event log"): + dispatch_once(ready_workspace) + assert len(_iteration_events(ready_workspace)) == 1 + + +def test_dispatch_result_names_the_object_and_the_record(ready_workspace): + result = dispatch_once(ready_workspace) + assert result["evidence"].endswith("evidence-iter1.json") + assert "objects" in result["object"] + + +def test_injected_verifier_dispatch_binds_its_own_bundle(ready_workspace): + dispatch_once(ready_workspace, verifier=lambda task, ws: VerifyOutcome(True, "injected")) + bundle = json.loads((ready_workspace / ".loop" / "artifacts" / "verify-iter1.json") + .read_text(encoding="utf-8")) + entry = next(e for e in _iteration_events(ready_workspace)[-1]["artifact_hashes"] + if e["path"].endswith("verify-iter1.json")) + assert bundle["verifier"]["source"] == "injected_callable" + assert entry["sha256"] == hashlib.sha256( + (ready_workspace / entry["path"]).read_bytes()).hexdigest() + + +def test_two_dispatches_bind_distinct_artifact_sets(two_task_workspace): + dispatch_once(two_task_workspace) + dispatch_once(two_task_workspace) + first, second = _iteration_events(two_task_workspace)[:2] + assert {e["path"] for e in first["artifact_hashes"]} != {e["path"] for e in second["artifact_hashes"]} + + +def test_terminal_written_carries_no_artifact_hashes(ready_workspace): + dispatch_once(ready_workspace) + dispatch_once(ready_workspace) + terminal = [e for e in _events(ready_workspace) if e["type"] == "terminal_written"] + assert terminal and terminal[-1]["artifact_hashes"] == [] diff --git a/scripts/test_loop_simulate_cli.py b/scripts/test_loop_simulate_cli.py index 31d0251..49052f3 100644 --- a/scripts/test_loop_simulate_cli.py +++ b/scripts/test_loop_simulate_cli.py @@ -83,7 +83,7 @@ def test_simulate_on_execute_task_with_undeclared_or_unparseable_verify_command_ def test_simulate_on_execute_task_with_all_tasks_done_predicts_would_write_terminal_with_exact_payload_preview(tmp_path): w, _ = _ws(tmp_path, [_task(status="done")]); r = simulate_run(w)["would"] - assert r["action"] == "would_write_terminal" and r["predicted_terminal"] == {"state": "Succeeded", "criteria_met": {"T-1": True}, "evidence": ["RUNLOG.md"], "false_completion": False, "completion_policy": {"mode": "all_required"}, "iteration_id": 4} + assert r["action"] == "would_write_terminal" and r["predicted_terminal"] == {"state": "Succeeded", "criteria_met": {"T-1": True}, "evidence": ["RUNLOG.md"], "false_completion": False, "completion_policy": {"mode": "all_required"}, "iteration_id": 4, "reason": "tasks with no evidence record: T-1"} def test_simulate_on_execute_task_with_unsatisfiable_dependency_predicts_would_block_with_null_refusal_reason(tmp_path): diff --git a/scripts/test_metrics_verifier_source.py b/scripts/test_metrics_verifier_source.py new file mode 100644 index 0000000..55f4af7 --- /dev/null +++ b/scripts/test_metrics_verifier_source.py @@ -0,0 +1,100 @@ +"""An injected-callable verdict is a caller's self-report, not gate evidence.""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from metrics import compute_metrics # noqa: E402 + +_ROOT = Path(__file__).resolve().parent.parent + + +def _bundle(source=None, *, green=True, task="T1", iteration_id=1): + body = {"iteration_id": iteration_id, "task": task, + "outcome": "PASS" if green else "FAIL", "passed": green, "score": 1.0 if green else 0.0} + if source is not None: + body["verifier"] = {"by": "loop.run", "source": source} + return body + + +def _repair(*, before, after, iteration_id="iter-001"): + return {"schema": "loop-engineer/repair@1", "iteration_id": iteration_id, "attempt": 1, + "failure_mode": "deterministic-fail", "hypothesis": "h", "repair_action": "a", + "verification_before": {"verify_full": "FAIL", "metric": "score", + "failing": ["x"], "score": before}, + "verification_after": {"verify_full": "PASS", "metric": "score", + "failing": [], "score": after}, + "remaining_delta": "none", "productive": True} + + +def _ws(tmp_path, bundles, repairs=()): + target = tmp_path / "workspace" + (target / ".loop" / "artifacts").mkdir(parents=True, exist_ok=True) + (target / "RUNLOG.md").write_text( + "# RUNLOG\n\n## Iteration 1 — 2026-07-25\n\n### Outcome\n\n`task_passed`\n", + encoding="utf-8") + (target / ".loop" / "state.json").write_text(json.dumps( + {"schema": "loop-engineer/state@1", "state": "execute-task", "iteration_id": 1, + "terminal_state": None}), encoding="utf-8") + for name, body in bundles: + (target / ".loop" / "artifacts" / name).write_text(json.dumps(body), encoding="utf-8") + if repairs: + (target / ".loop" / "repair").mkdir(parents=True, exist_ok=True) + for name, body in repairs: + (target / ".loop" / "repair" / name).write_text(json.dumps(body), encoding="utf-8") + return target + + +def test_a_declared_command_bundle_is_gate_evidence(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("declared_command"))]) + assert compute_metrics(target)["false_completion_rate"] == 0.0 + + +def test_an_injected_callable_bundle_is_not_gate_evidence(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable"))]) + assert compute_metrics(target)["provenance"]["injected_verifier_bundles"] == ["verify-iter1.json"] + + +def test_an_injected_only_claim_is_a_false_completion(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable"))]) + assert compute_metrics(target)["false_completion_rate"] == 1.0 + + +def test_a_legacy_bundle_without_a_verifier_block_still_counts(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle(None))]) + metrics = compute_metrics(target) + assert metrics["false_completion_rate"] == 0.0 + assert metrics["provenance"]["injected_verifier_bundles"] == [] + + +def test_excluded_bundles_are_named_under_provenance(tmp_path): + target = _ws(tmp_path, [("verify-iter1.json", _bundle("injected_callable")), + ("verify-iter2.json", _bundle("declared_command", iteration_id=1))]) + assert compute_metrics(target)["provenance"]["injected_verifier_bundles"] == ["verify-iter1.json"] + + +def test_injected_bundles_do_not_anchor_a_repair(tmp_path): + """An entirely self-reported red-to-green pair must not corroborate a repair. + + The repair record is load-bearing: without one on disk, ``repair_productivity`` + is ``None`` for the unrelated reason that nothing was validated, and the test + passes against unchanged code. With one, the pair anchors before this change + and cannot after it. + + It deliberately does not claim more than it proves. An unanchored record still + counts toward plain-mode ``repair_productivity``; only ``--baseline`` refuses. + """ + target = _ws(tmp_path, + [("verify-iter1.json", _bundle("injected_callable", green=False)), + ("verify-iter2.json", _bundle("injected_callable"))], + repairs=[("iter-001.json", _repair(before=0.0, after=1.0))]) + assert compute_metrics(target)["provenance"]["unanchored_records"] == [ + ".loop/repair/iter-001.json"] + + +def test_the_shipped_examples_keep_their_published_metrics(): + metrics = compute_metrics(_ROOT / "examples" / "flaky-test-triage") + assert metrics["false_completion_rate"] == 0.0 + assert metrics["repair_productivity"] == 1.0 diff --git a/scripts/test_verifier_identity.py b/scripts/test_verifier_identity.py index 53ccfef..b6e20f5 100644 --- a/scripts/test_verifier_identity.py +++ b/scripts/test_verifier_identity.py @@ -67,6 +67,20 @@ def test_code_digest_says_unresolvable_when_resolution_raises(tmp_path): assert verifier_code_digest("./a.sh", tmp_path) == (None, "unresolvable") +def test_code_digest_says_unresolvable_when_the_WORKSPACE_ROOT_cannot_be_resolved(tmp_path): + """The workspace root is resolved inside the SAME try as argv[0], and must stay there. + + Both interpreter behaviours land on `unresolvable`: on <=3.12 `Path.resolve()` raises + pathlib's RuntimeError for a symlink loop, while 3.13 returns the loop unresolved and + `os.stat` raises ELOOP as an OSError. Narrowing that except boundary to argv[0] alone + is exactly how Slice 2's Task-1 Critical was born, so the root variant is pinned too. + """ + root, other = tmp_path / "root-a", tmp_path / "root-b" + root.symlink_to(other) + other.symlink_to(root) # ELOOP on resolving the workspace root itself + assert verifier_code_digest("./scripts/verify-fast.sh", root) == (None, "unresolvable") + + def test_code_digest_is_null_and_explained_when_the_file_cannot_be_read(tmp_path, monkeypatch): _script(tmp_path, "scripts/verify-fast.sh") monkeypatch.setattr(verifier, "_digest_file", lambda path: None) diff --git a/scripts/test_verify_evidence_object_collision.py b/scripts/test_verify_evidence_object_collision.py new file mode 100644 index 0000000..99b4ac5 --- /dev/null +++ b/scripts/test_verify_evidence_object_collision.py @@ -0,0 +1,67 @@ +"""The object-store collision handler must not raise a raw OSError. + +``write_verify_evidence`` places the content-addressed object with a create-once hard +link and treats an existing object as idempotent success once its bytes match. The +match check reads the existing object from INSIDE the ``except FileExistsError`` +handler, where the sibling ``except OSError`` cannot reach it — so an +``IsADirectoryError`` or ``PermissionError`` at the object path escaped the writer +untyped, past the API's typed-error contract. +""" +from __future__ import annotations + +import json + +import pytest + +from loop import emit +from loop.evidence import artifact_object_path +from loop.verifier import injected_verifier_identity + +_TASK = {"id": "T-1", "title": "T-1", "status": "pending", "criterion_ref": "C-1", + "verify": "./scripts/verify-fast.sh", "depends_on": [], "attempts": 0, "evidence": None} + + +def _contract(tmp_path, name="workspace"): + workspace = tmp_path / name + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": [_TASK]}), encoding="utf-8") + return workspace + + +def _built(workspace): + return emit.build_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + + +def test_an_unreadable_existing_object_is_a_typed_emit_error(tmp_path): + workspace = _contract(tmp_path) + built = _built(workspace) + # A DIRECTORY at the object path: the hard link refuses it (FileExistsError) and the + # collision read then raises IsADirectoryError from inside the handler. + built.object_path.mkdir(parents=True) + with pytest.raises(emit.EmitError, match="cannot be read"): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + + +def test_a_colliding_object_with_different_bytes_is_still_the_refusal_it_was(tmp_path): + """Control: the pre-existing typed refusal for a genuine digest collision is unchanged.""" + workspace = _contract(tmp_path) + built = _built(workspace) + built.object_path.parent.mkdir(parents=True, exist_ok=True) + built.object_path.write_text("different bytes\n", encoding="utf-8") + with pytest.raises(emit.EmitError, match="refusing to overwrite"): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + + +def test_an_identical_existing_object_is_idempotent_success(tmp_path): + """Control: a re-run of the same dispatch still writes cleanly.""" + workspace = _contract(tmp_path) + first = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + second = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_TASK, + passed=True, code_identity=injected_verifier_identity()) + assert first["sha256"] == second["sha256"] + assert artifact_object_path(workspace, first["sha256"]).is_file() diff --git a/scripts/test_verify_evidence_objects.py b/scripts/test_verify_evidence_objects.py new file mode 100644 index 0000000..24fa4e3 --- /dev/null +++ b/scripts/test_verify_evidence_objects.py @@ -0,0 +1,150 @@ +"""build_verify_evidence purity, the content-addressed object, and the bound digest set.""" +from __future__ import annotations + +import hashlib +import json + +import pytest + +from loop import emit +from loop.evidence import artifact_object_path +from loop.verifier import executed_verifier_identity + + +def _task(**overrides): + base = {"id": "T-1", "title": "t", "status": "pending", "criterion_ref": "C-1", + "verify": "./scripts/verify-fast.sh", "depends_on": [], "attempts": 0, "evidence": None} + base.update(overrides) + return base + + +def _ws(tmp_path): + workspace = tmp_path / "workspace" + emit.open_contract(workspace) + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + return workspace + + +def _tree(workspace): + return {str(p.relative_to(workspace)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in workspace.rglob("*") if p.is_file()} + + +def _build(workspace, *, iteration_id=1, task=None, passed=True, **kwargs): + task = task or _task() + kwargs.setdefault("code_identity", executed_verifier_identity(task["verify"], workspace)) + return emit.build_verify_evidence(workspace, run_id="run-1", iteration_id=iteration_id, + task=task, passed=passed, **kwargs) + + +def test_build_writes_nothing(tmp_path): + workspace = _ws(tmp_path) + before = _tree(workspace) + _build(workspace) + assert _tree(workspace) == before + + +def test_build_and_write_agree_on_every_byte(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + written = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), + built=built) + assert written["bundle"].read_text(encoding="utf-8") == built.bundle_text + assert written["evidence"].read_text(encoding="utf-8") == built.record_text + + +def test_write_places_the_content_addressed_object(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + written = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + assert written["object"].is_file() + assert hashlib.sha256(written["object"].read_bytes()).hexdigest() == built.sha256 + + +def test_object_path_matches_artifact_object_path(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + assert built.object_path == artifact_object_path(workspace, built.sha256) + + +def test_artifact_hashes_cover_bundle_object_and_record(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + by_path = {entry["path"]: entry["sha256"] for entry in built.artifact_hashes} + assert len(built.artifact_hashes) == 3 + assert by_path[".loop/artifacts/verify-iter1.json"] == built.sha256 + assert by_path[".loop/evidence/evidence-iter1.json"] == built.record_sha256 + assert by_path[built.object_path.relative_to(workspace).as_posix()] == built.sha256 + + +def test_artifact_hashes_entries_are_workspace_relative_posix(tmp_path): + built = _build(_ws(tmp_path)) + for entry in built.artifact_hashes: + assert not entry["path"].startswith("/") and "\\" not in entry["path"] + assert len(entry["sha256"]) == 64 + + +def test_record_sha256_still_commits_to_the_bundle_bytes(tmp_path): + built = _build(_ws(tmp_path)) + record = json.loads(built.record_text) + assert record["sha256"] == hashlib.sha256(built.bundle_text.encode("utf-8")).hexdigest() + + +def test_rewriting_the_bundle_leaves_the_object_intact(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + written = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + written["bundle"].write_text('{"outcome": "PASS", "passed": true}', encoding="utf-8") + assert hashlib.sha256(written["object"].read_bytes()).hexdigest() == built.sha256 + + +def test_a_second_identical_build_reuses_the_object_without_error(tmp_path): + workspace = _ws(tmp_path) + identity = executed_verifier_identity(_task()["verify"], workspace) + first = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, code_identity=identity) + second = emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, + task=_task(), passed=True, code_identity=identity) + assert first["object"] == second["object"] and first["sha256"] == second["sha256"] + + +def test_an_object_collision_with_different_bytes_raises_emit_error(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + built.object_path.parent.mkdir(parents=True, exist_ok=True) + built.object_path.write_text("not the bundle", encoding="utf-8") + with pytest.raises(emit.EmitError, match="object store"): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_task(), + passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + + +def test_a_non_canonicalizable_task_raises_a_typed_emit_error_at_build_time(tmp_path): + workspace = _ws(tmp_path) + with pytest.raises(emit.EmitError, match="canonical"): + _build(workspace, task=_task(criterion_ref=float("nan"))) + + +def test_a_failed_record_write_leaves_no_metrics_visible_bundle_and_no_partial_object(tmp_path): + workspace = _ws(tmp_path) + built = _build(workspace) + (workspace / ".loop" / "evidence").mkdir(parents=True, exist_ok=True) + (workspace / ".loop" / "evidence" / "evidence-iter1.json").mkdir() # write must fail + with pytest.raises((OSError, emit.EmitError)): + emit.write_verify_evidence(workspace, run_id="run-1", iteration_id=1, task=_task(), + passed=True, + code_identity=executed_verifier_identity( + _task()["verify"], workspace), built=built) + assert not list((workspace / ".loop" / "artifacts").glob("verify-*.json")) diff --git a/scripts/test_verify_evidence_writer.py b/scripts/test_verify_evidence_writer.py index 3d946b0..4bb5213 100644 --- a/scripts/test_verify_evidence_writer.py +++ b/scripts/test_verify_evidence_writer.py @@ -146,7 +146,10 @@ def test_rewrite_of_the_same_iteration_is_idempotent_in_path(tmp_path): first, second = _write(workspace), _write(workspace) assert first["bundle"] == second["bundle"] and first["evidence"] == second["evidence"] assert first["sha256"] == second["sha256"] - assert [p.name for p in sorted((workspace / ".loop" / "artifacts").iterdir())] == ["verify-iter5.json"] + # Files only: the content-addressed object store is a declared subtree under + # artifacts/, not litter. A duplicate bundle or an orphan .staged still fails. + assert [p.name for p in sorted((workspace / ".loop" / "artifacts").iterdir()) + if p.is_file()] == ["verify-iter5.json"] def test_writer_refuses_a_workspace_with_no_contract(tmp_path):