From de494c4f4ebeb0b205c868cc28ac49007b8e4781 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 22:55:27 +0000 Subject: [PATCH] Ledger: restore records the migration invalidated for missing evidence (#39) The migration invalidates a committed record whose bundle is neither on disk nor in the archive, and invalidating is permanent. But evidence that was never archived (a pre-#16 ledger) may still exist on disk. - The migration's error now says why: "evidence missing" (neither on disk nor archived) or "evidence mismatched" (anything else). - stele.ledger.reverify (reverify_record, and python -m stele.ledger.reverify --artifact-dir DIR) seals such a record again from files that hash as recorded. It is an explicit operator step, never part of migration, and refuses unless the record was invalidated by the migration for missing evidence (its error says so and its only event is the migration's import), has no deliveries, and the files match the manifest. Mismatched evidence, operator or replay invalidations, and earlier migrations' unlabelled errors stay invalidated. - A chained record.reverified event is written in the same transaction; verify_ledger replays it as sealed. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_013p3FU6gUJ1MGinnmEgvHsm --- docs/ledger.md | 24 +++- stele/ledger/events.py | 4 +- stele/ledger/migration.py | 18 ++- stele/ledger/reverify.py | 148 +++++++++++++++++++++ tests/test_reverify.py | 263 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 451 insertions(+), 6 deletions(-) create mode 100644 stele/ledger/reverify.py create mode 100644 tests/test_reverify.py diff --git a/docs/ledger.md b/docs/ledger.md index 6f5a659..6a7b3ae 100644 --- a/docs/ledger.md +++ b/docs/ledger.md @@ -31,7 +31,7 @@ This is the one definition of ledger states. Other docs and code link here. | `pending` | The bundle is hashed and recorded; it is not yet archived. | `seal`, `fail`, `invalidate` | | `sealed` | Every artifact and the bundle's tree object are stored in the evidence archive and were verified against the recorded manifest. | `invalidate` | | `failed` | The parser failed, a check before sealing failed, or sealing failed. Never becomes sealed. | terminal | -| `invalidated` | Withdrawn after the fact (see [replay.md](replay.md)). The record is kept. | terminal | +| `invalidated` | Withdrawn after the fact (see [replay.md](replay.md)). The record is kept. | terminal, except a record the migration invalidated for missing evidence, which `reverify` can seal again (below) | - **Sealed is integrity, not delivery.** A sealed record says nothing about whether any downstream target has received it. Delivery is a separate set @@ -151,12 +151,30 @@ Migrating a pre-#12 (version 0) ledger: |------------|---------| | `pending`, `failed`, `invalidated` | Same state | | `committed`, bundle archivable and matching its manifest (from disk, or already in the archive) | `sealed` | -| `committed`, bundle missing, changed or unsafe | `invalidated`, with the reason in `error` — the ledger can no longer vouch for it | +| `committed`, bundle missing, changed or unsafe | `invalidated`, with the reason in `error` — the ledger can no longer vouch for it. The error says `evidence missing` (neither on disk nor archived) or `evidence mismatched` (anything else) | | `source_hash` that is the digest of exactly one Snapshot in the archive | Kept, with its `source_kind` | | any other `source_hash` | Moved to `legacy_source_hash` | | `source_path` | Kept, and recorded as a Source in the archive | | parser identity and config | Unknown: `NULL` | +**Restoring missing evidence.** Invalidating is permanent, but evidence the +archive never held (a pre-#16 ledger) may still exist on disk. After migration, +an operator who has the files can seal such a record again: + +```bash +python -m stele.ledger.reverify ledger.db archive/ --artifact-dir /where/the/files/are +``` + +This is never done by migration, and it is narrow. The record must have been +invalidated *by the migration* for *missing* evidence: its error says so and +its only event is the migration's import, so a record an operator or a replay +invalidated stays invalidated, as does one whose evidence was `mismatched`. It +must have no deliveries (invalidating withdrew the delivered data). The files +are hashed against the recorded manifest and archived exactly as sealing does; +any difference leaves the record invalidated. On success the record is sealed, +its error cleared, and a `record.reverified` event is chained, so the history +reads: imported as invalidated, reverified. + Two records for one `run_id` cannot be represented; migration stops with `LedgerMigrationError` and changes nothing. A ledger newer than this Stele raises `LedgerSchemaError`. @@ -171,6 +189,7 @@ the two can't diverge: |---|---| | `record.created` | `create_pending`: run, artifact hash, input Snapshot, Source, parser identity and config, backend | | `record.sealed` / `record.failed` / `record.invalidated` | each state transition, with its error or reason | +| `record.reverified` | an operator restored a record the migration invalidated for missing evidence (below) | | `delivery.opened` / `delivery.event` | each delivery and each intent, receipt, failure or removal in the delivery log | | `replay.logged` | each replay and its outcome | | `*.imported` | the state found when a ledger was migrated to version 5 | @@ -217,6 +236,7 @@ if there is any problem. - `stele/ledger/delivery.py` — the delivery log (#13) - `tests/test_ledger.py` — state machine, hashing, per-run records - `tests/test_ledger_provenance.py` — Snapshot provenance, parser identity, sealing, migration +- `tests/test_reverify.py` — restoring records invalidated for missing evidence - `tests/test_event_log.py` — chaining, verification, tamper detection, anchors, migration, CLI ## Completion criteria diff --git a/stele/ledger/events.py b/stele/ledger/events.py index cbae4f8..b22000b 100644 --- a/stele/ledger/events.py +++ b/stele/ledger/events.py @@ -4,7 +4,8 @@ Every fact the ledger learns is appended here as an event, in the same database transaction as the change it describes, so the two cannot diverge: - record.created / record.sealed / record.failed / record.invalidated + record.created / record.sealed / record.failed / record.invalidated / + record.reverified delivery.opened / delivery.event (the #13 delivery log) replay.logged (the #14 replay log) *.imported (state found when a ledger was migrated) @@ -217,6 +218,7 @@ def ok(self) -> bool: "record.sealed": "sealed", "record.failed": "failed", "record.invalidated": "invalidated", + "record.reverified": "sealed", # an operator restored missing evidence (ledger.reverify) } diff --git a/stele/ledger/migration.py b/stele/ledger/migration.py index 487be65..64c760e 100644 --- a/stele/ledger/migration.py +++ b/stele/ledger/migration.py @@ -23,7 +23,10 @@ archive and verified against the recorded manifest (from disk, or from the archive if it already holds the bundle). A committed record whose bundle cannot be verified becomes INVALIDATED with the reason: the ledger can no - longer vouch for that content. + longer vouch for that content. The reason says whether the evidence was + missing (neither on disk nor archived) or mismatched; only a record with + missing evidence can be restored later, by ledger.reverify, from files that + still hash as recorded. - source_hash is kept only when it is the digest of exactly one Snapshot in the archive. Otherwise it moves to legacy_source_hash, which is unverified audit data and never used as provenance. @@ -55,6 +58,11 @@ ) +# Why a committed record was invalidated by migration (prefix of its error). +EVIDENCE_MISSING = "evidence missing" +EVIDENCE_MISMATCHED = "evidence mismatched" + + class LedgerMigrationError(Exception): """Raised when an existing ledger cannot be migrated; nothing is changed.""" @@ -162,9 +170,13 @@ def _migrate_row(old: dict, archive: BlobStore) -> dict: ) as exc: row["state"] = "invalidated" row["finalized_at"] = datetime.now(timezone.utc).isoformat() + # Missing evidence (neither on disk nor archived) can be restored + # by an operator who still has the files (ledger.reverify); + # anything else means the content is not what was recorded. + kind = EVIDENCE_MISSING if isinstance(exc, MissingArtifactError) else EVIDENCE_MISMATCHED row["error"] = ( - "INVALIDATED: migration — committed bundle could not be archived " - f"and verified: {exc!r}" + f"INVALIDATED: migration — {kind}: committed bundle could not be " + f"archived and verified: {exc!r}" ) else: row["state"] = "sealed" diff --git a/stele/ledger/reverify.py b/stele/ledger/reverify.py new file mode 100644 index 0000000..fa0c213 --- /dev/null +++ b/stele/ledger/reverify.py @@ -0,0 +1,148 @@ +""" +Restoring a record the migration invalidated for missing evidence (#39). + +Migrating a pre-#12 ledger invalidates a committed record whose bundle can be +neither archived from disk nor found in the archive. Often the evidence was +never archived (the archive did not exist yet) and the files are still +somewhere: invalidating is safe, but it is permanent, so such a record could +never be sealed again. + +reverify_record() is the explicit operator step that restores it. It never +happens during migration, and it is narrow: + +- the record must be INVALIDATED *by the migration*, for *missing* evidence. + Its error says so, and its event history is exactly the migration's import + of it: an invalidation by an operator or a divergent replay has a + record.invalidated event (or another error) and stays invalidated, as does + a record whose evidence was found to be different (mismatched); +- it must have no deliveries, since invalidation withdraws delivered data; +- the files in the given directory are hashed against the recorded manifest + and archived exactly as sealing does. Any difference, or a missing file, + leaves the record INVALIDATED. + +On success the record is SEALED again, its error cleared, and a +`record.reverified` event is appended to the hash chain in the same +transaction; the earlier events stay, so the history reads: imported as +invalidated, reverified. + + python -m stele.ledger.reverify --artifact-dir DIR +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from ..archive import BlobStore +from .events import EventLog +from .migration import EVIDENCE_MISSING +from .models import ArtifactRecord, ArtifactState +from .store import ( + ArtifactDriftError, + InvalidStateTransitionError, + LedgerStore, + MissingArtifactError, + RecordNotFoundError, + archive_bundle, +) + +_MIGRATION_PREFIX = "INVALIDATED: migration — " + + +class ReverifyRefused(Exception): + """The record cannot be restored; nothing was changed.""" + + +def reverifiable_problem(ledger: LedgerStore, record: ArtifactRecord) -> str | None: + """Why the record may not be reverified; None if it may.""" + if record.state is not ArtifactState.INVALIDATED: + return f"the record is {record.state.value}, not invalidated" + error = record.error or "" + if not error.startswith(f"{_MIGRATION_PREFIX}{EVIDENCE_MISSING}:"): + return "it was not invalidated by the migration for missing evidence" + log = EventLog(ledger) + try: + events = log.for_subject(record.record_id) + finally: + log.close() + if [e.kind for e in events] != ["record.imported"] or events[0].body.get("state") != "invalidated": + return "its history is not just the migration's import, so it was invalidated for another reason" + deliveries = ledger._conn.execute( + "SELECT COUNT(*) FROM deliveries WHERE record_id=?", (record.record_id,) + ).fetchone()[0] + if deliveries: + return "it has deliveries, and invalidating withdrew the delivered data" + return None + + +def reverify_record( + ledger: LedgerStore, record_id: str, artifact_dir: Path | str | None = None +) -> ArtifactRecord: + """Seal an invalidated record again from files that hash as recorded. + + artifact_dir defaults to the directory the record names. Raises + ReverifyRefused (nothing changed) for a record that may not be restored + or whose files do not match. + """ + try: + record = ledger.get(record_id) + except RecordNotFoundError: + raise ReverifyRefused(f"no record {record_id}") from None + problem = reverifiable_problem(ledger, record) + if problem is not None: + raise ReverifyRefused(f"record {record_id} cannot be reverified: {problem}") + directory = Path(artifact_dir) if artifact_dir is not None else Path(record.artifact_dir) + try: + archive_bundle(ledger.archive, directory, record.artifact_manifest, record.artifact_hash) + except (MissingArtifactError, ArtifactDriftError) as exc: + raise ReverifyRefused( + f"record {record_id}: the files in {directory} are not the recorded bundle: {exc}" + ) from exc + try: + ledger._transition( + record_id, + from_states=(ArtifactState.INVALIDATED,), + action="reverify", + set_sql="state='sealed', finalized_at=?, error=NULL", + params=(_now_iso(),), + event=("record.reverified", { + "artifact_hash": record.artifact_hash, + "previous_error": record.error, + }), + ) + except InvalidStateTransitionError as exc: + raise ReverifyRefused(str(exc)) from exc + return ledger.get(record_id) + + +def _now_iso() -> str: + from datetime import datetime, timezone + + return datetime.now(timezone.utc).isoformat() + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + prog="python -m stele.ledger.reverify", + description="Seal a migration-invalidated record again from files that hash as recorded.", + ) + ap.add_argument("ledger", type=Path) + ap.add_argument("archive", type=Path) + ap.add_argument("record_id") + ap.add_argument("--artifact-dir", type=Path, default=None, + help="where the record's files are now (default: the directory it names)") + args = ap.parse_args(argv) + ledger = LedgerStore(args.ledger, BlobStore(args.archive)) + try: + record = reverify_record(ledger, args.record_id, args.artifact_dir) + except ReverifyRefused as exc: + print(str(exc), file=sys.stderr) + return 1 + finally: + ledger.close() + print(f"record {record.record_id} is sealed again (artifact_hash {record.artifact_hash})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_reverify.py b/tests/test_reverify.py new file mode 100644 index 0000000..3f3954e --- /dev/null +++ b/tests/test_reverify.py @@ -0,0 +1,263 @@ +""" +Restoring records the migration invalidated for missing evidence (#39). + + 1. The migration says *why* it invalidated a committed record: missing + evidence (neither on disk nor archived) or mismatched evidence. + 2. reverify_record() seals a missing-evidence record again from files that + hash as recorded, logs record.reverified, and the ledger still verifies. + 3. It refuses everything else and changes nothing: mismatched evidence, + files that differ, records invalidated by an operator or a replay, + records with deliveries, records that are not invalidated. + 4. The CLI does the same. +""" +from __future__ import annotations + +import json +import shutil +import sqlite3 +import uuid +from pathlib import Path + +import pytest + +from stele.ledger.events import EventLog, verify_ledger +from stele.ledger.hashing import build_manifest, sha256_manifest +from stele.ledger.models import ArtifactState, ParserIdentity +from stele.ledger.reverify import ReverifyRefused, main as cli, reverify_record +from tests.ledger_helpers import open_ledger +from tests.test_ledger_provenance import _V0_DDL + + +@pytest.fixture +def v0(tmp_path: Path): + """A pre-#12 ledger; add(...) returns (record_id, output directory).""" + db = tmp_path / "ledger.db" + conn = sqlite3.connect(db) + conn.executescript(_V0_DDL) + counter = iter(range(100)) + + def add(state: str, *, keep_files: bool = True, error: str | None = None, + content: str | None = None) -> tuple[str, Path]: + n = next(counter) + out = tmp_path / f"out-{n}" + out.mkdir() + (out / "r.json").write_text(content or f"output {n}") + (out / "s.txt").write_text(f"second file {n}") + manifest = build_manifest(out, [out / "r.json", out / "s.txt"]) + if not keep_files: + shutil.rmtree(out) + record_id = str(uuid.uuid4()) + conn.execute( + "INSERT INTO artifact_records VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (record_id, str(uuid.uuid4()), None, None, str(out), json.dumps(manifest), + sha256_manifest(manifest), state, f"2026-06-26T00:00:{n:02d}+00:00", + None if state == "pending" else "2026-06-26T01:00:00+00:00", error), + ) + conn.commit() + return record_id, out + + yield db, add + conn.close() + + +def _restore(out: Path, n: int) -> None: + """Put back the files the v0 fixture wrote for output n.""" + out.mkdir(exist_ok=True) + (out / "r.json").write_text(f"output {n}") + (out / "s.txt").write_text(f"second file {n}") + + +def _kinds(ledger, record_id: str) -> list[str]: + log = EventLog(ledger) + try: + return [e.kind for e in log.for_subject(record_id)] + finally: + log.close() + + +# --------------------------------------------------------------------------- +# 1 — the migration says why +# --------------------------------------------------------------------------- + +def test_migration_distinguishes_missing_from_mismatched_evidence(v0, tmp_path) -> None: + db, add = v0 + missing, _ = add("committed", keep_files=False) + mismatched, out = add("committed") + (out / "r.json").write_text("not what was recorded") + + ledger = open_ledger(db) + assert ledger.get(missing).error.startswith("INVALIDATED: migration — evidence missing:") + assert ledger.get(mismatched).error.startswith("INVALIDATED: migration — evidence mismatched:") + + +# --------------------------------------------------------------------------- +# 2 — restoring +# --------------------------------------------------------------------------- + +def test_files_that_hash_as_recorded_seal_the_record_again(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + before = ledger.get(record_id) + assert before.state is ArtifactState.INVALIDATED + + _restore(out, 0) + record = reverify_record(ledger, record_id) + + assert record.state is ArtifactState.SEALED + assert record.error is None + assert record.artifact_hash == before.artifact_hash + assert ledger.archive.read_tree(record.artifact_hash) == record.artifact_manifest + assert _kinds(ledger, record_id) == ["record.imported", "record.reverified"] + report = verify_ledger(ledger) + assert report.ok, (report.chain.problems, report.problems) + + +def test_files_can_be_given_from_another_directory(v0, tmp_path) -> None: + db, add = v0 + record_id, _ = add("committed", keep_files=False) + elsewhere = tmp_path / "recovered" + _restore(elsewhere, 0) + ledger = open_ledger(db) + + assert reverify_record(ledger, record_id, elsewhere).state is ArtifactState.SEALED + assert verify_ledger(ledger).ok + + +def test_a_restored_record_is_an_ordinary_sealed_record(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + _restore(out, 0) + reverify_record(ledger, record_id) + shutil.rmtree(out) # the archive alone vouches for it now + reopened = open_ledger(db) + assert reopened.get(record_id).state is ArtifactState.SEALED + assert verify_ledger(reopened).ok + with pytest.raises(ReverifyRefused, match="not invalidated"): + reverify_record(reopened, record_id) + + +# --------------------------------------------------------------------------- +# 3 — everything else is refused, and nothing changes +# --------------------------------------------------------------------------- + +def _assert_untouched(ledger, record_id: str, events: list[str]) -> None: + assert ledger.get(record_id).state is ArtifactState.INVALIDATED + assert _kinds(ledger, record_id) == events + assert verify_ledger(ledger).ok + + +def test_mismatched_evidence_is_never_restored(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed") + (out / "r.json").write_text("not what was recorded") + ledger = open_ledger(db) + _restore(out, 0) # even with the right files back + with pytest.raises(ReverifyRefused, match="not invalidated by the migration for missing"): + reverify_record(ledger, record_id) + _assert_untouched(ledger, record_id, ["record.imported"]) + + +@pytest.mark.parametrize("damage", ["changed", "missing_file", "extra_only"]) +def test_files_that_differ_leave_the_record_invalidated(v0, tmp_path, damage) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + _restore(out, 0) + if damage == "changed": + (out / "s.txt").write_text("someone edited this") + elif damage == "missing_file": + (out / "s.txt").unlink() + else: + (out / "r.json").unlink() + (out / "s.txt").unlink() + (out / "other.txt").write_text("not in the manifest") + with pytest.raises(ReverifyRefused, match="not the recorded bundle"): + reverify_record(ledger, record_id) + _assert_untouched(ledger, record_id, ["record.imported"]) + + +def test_a_record_invalidated_by_an_operator_stays_invalidated(v0, tmp_path) -> None: + """Even with the migration's own words as the note, the history gives it away.""" + db, add = v0 + ledger = open_ledger(db) + out = tmp_path / "fresh" + out.mkdir() + (out / "a.txt").write_text("a") + pending = ledger.create_pending( + run_id=str(uuid.uuid4()), artifact_dir=out, artifact_paths=[out / "a.txt"], + parser=ParserIdentity("p", "1"), parser_config={}, + ) + ledger.seal(pending.record_id) + ledger.invalidate(pending.record_id, "migration — evidence missing: by hand") + with pytest.raises(ReverifyRefused, match="invalidated for another reason"): + reverify_record(ledger, pending.record_id, out) + assert ledger.get(pending.record_id).state is ArtifactState.INVALIDATED + + +def test_the_pre_reverify_error_text_is_not_trusted(v0, tmp_path) -> None: + """Invalidated by an earlier migration that did not say why: stays put.""" + db, add = v0 + record_id, out = add( + "invalidated", + error="INVALIDATED: migration — committed bundle could not be archived and verified: " + "MissingArtifactError('x')", + ) + ledger = open_ledger(db) + _restore(out, 0) + with pytest.raises(ReverifyRefused, match="not invalidated by the migration for missing"): + reverify_record(ledger, record_id) + _assert_untouched(ledger, record_id, ["record.imported"]) + + +def test_a_record_with_deliveries_is_refused(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + conn = sqlite3.connect(db) + conn.execute( + "INSERT INTO deliveries (dispatch_id, record_id, target_kind, target, created_at) " + "VALUES ('d1', ?, 'T', '{}', '2026-06-26T02:00:00+00:00')", (record_id,), + ) + conn.commit() + conn.close() + _restore(out, 0) + with pytest.raises(ReverifyRefused, match="has deliveries"): + reverify_record(ledger, record_id) + assert ledger.get(record_id).state is ArtifactState.INVALIDATED + + +@pytest.mark.parametrize("state", ["committed", "pending", "failed"]) +def test_only_invalidated_records_are_reverified(v0, tmp_path, state) -> None: + db, add = v0 + record_id, _ = add(state, error="x" if state == "failed" else None) + ledger = open_ledger(db) + with pytest.raises(ReverifyRefused, match="not invalidated"): + reverify_record(ledger, record_id) + + +def test_unknown_record(v0) -> None: + ledger = open_ledger(v0[0]) + with pytest.raises(ReverifyRefused, match="no record"): + reverify_record(ledger, str(uuid.uuid4())) + + +# --------------------------------------------------------------------------- +# 4 — CLI +# --------------------------------------------------------------------------- + +def test_cli(v0, tmp_path, capsys) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + archive = str(ledger.archive.root) + ledger.close() + + assert cli([str(db), archive, record_id]) == 1 # files are still gone + assert "not the recorded bundle" in capsys.readouterr().err + _restore(out, 0) + assert cli([str(db), archive, record_id, "--artifact-dir", str(out)]) == 0 + assert "sealed again" in capsys.readouterr().out + assert cli([str(db), archive, record_id]) == 1 # already sealed + assert "not invalidated" in capsys.readouterr().err