diff --git a/docs/ledger.md b/docs/ledger.md index 6f5a659..6a7b3ae 100644 --- a/docs/ledger.md +++ b/docs/ledger.md @@ -31,7 +31,7 @@ This is the one definition of ledger states. Other docs and code link here. | `pending` | The bundle is hashed and recorded; it is not yet archived. | `seal`, `fail`, `invalidate` | | `sealed` | Every artifact and the bundle's tree object are stored in the evidence archive and were verified against the recorded manifest. | `invalidate` | | `failed` | The parser failed, a check before sealing failed, or sealing failed. Never becomes sealed. | terminal | -| `invalidated` | Withdrawn after the fact (see [replay.md](replay.md)). The record is kept. | terminal | +| `invalidated` | Withdrawn after the fact (see [replay.md](replay.md)). The record is kept. | terminal, except a record the migration invalidated for missing evidence, which `reverify` can seal again (below) | - **Sealed is integrity, not delivery.** A sealed record says nothing about whether any downstream target has received it. Delivery is a separate set @@ -151,12 +151,30 @@ Migrating a pre-#12 (version 0) ledger: |------------|---------| | `pending`, `failed`, `invalidated` | Same state | | `committed`, bundle archivable and matching its manifest (from disk, or already in the archive) | `sealed` | -| `committed`, bundle missing, changed or unsafe | `invalidated`, with the reason in `error` — the ledger can no longer vouch for it | +| `committed`, bundle missing, changed or unsafe | `invalidated`, with the reason in `error` — the ledger can no longer vouch for it. The error says `evidence missing` (neither on disk nor archived) or `evidence mismatched` (anything else) | | `source_hash` that is the digest of exactly one Snapshot in the archive | Kept, with its `source_kind` | | any other `source_hash` | Moved to `legacy_source_hash` | | `source_path` | Kept, and recorded as a Source in the archive | | parser identity and config | Unknown: `NULL` | +**Restoring missing evidence.** Invalidating is permanent, but evidence the +archive never held (a pre-#16 ledger) may still exist on disk. After migration, +an operator who has the files can seal such a record again: + +```bash +python -m stele.ledger.reverify ledger.db archive/ --artifact-dir /where/the/files/are +``` + +This is never done by migration, and it is narrow. The record must have been +invalidated *by the migration* for *missing* evidence: its error says so and +its only event is the migration's import, so a record an operator or a replay +invalidated stays invalidated, as does one whose evidence was `mismatched`. It +must have no deliveries (invalidating withdrew the delivered data). The files +are hashed against the recorded manifest and archived exactly as sealing does; +any difference leaves the record invalidated. On success the record is sealed, +its error cleared, and a `record.reverified` event is chained, so the history +reads: imported as invalidated, reverified. + Two records for one `run_id` cannot be represented; migration stops with `LedgerMigrationError` and changes nothing. A ledger newer than this Stele raises `LedgerSchemaError`. @@ -171,6 +189,7 @@ the two can't diverge: |---|---| | `record.created` | `create_pending`: run, artifact hash, input Snapshot, Source, parser identity and config, backend | | `record.sealed` / `record.failed` / `record.invalidated` | each state transition, with its error or reason | +| `record.reverified` | an operator restored a record the migration invalidated for missing evidence (below) | | `delivery.opened` / `delivery.event` | each delivery and each intent, receipt, failure or removal in the delivery log | | `replay.logged` | each replay and its outcome | | `*.imported` | the state found when a ledger was migrated to version 5 | @@ -217,6 +236,7 @@ if there is any problem. - `stele/ledger/delivery.py` — the delivery log (#13) - `tests/test_ledger.py` — state machine, hashing, per-run records - `tests/test_ledger_provenance.py` — Snapshot provenance, parser identity, sealing, migration +- `tests/test_reverify.py` — restoring records invalidated for missing evidence - `tests/test_event_log.py` — chaining, verification, tamper detection, anchors, migration, CLI ## Completion criteria diff --git a/stele/ledger/events.py b/stele/ledger/events.py index cbae4f8..b22000b 100644 --- a/stele/ledger/events.py +++ b/stele/ledger/events.py @@ -4,7 +4,8 @@ Every fact the ledger learns is appended here as an event, in the same database transaction as the change it describes, so the two cannot diverge: - record.created / record.sealed / record.failed / record.invalidated + record.created / record.sealed / record.failed / record.invalidated / + record.reverified delivery.opened / delivery.event (the #13 delivery log) replay.logged (the #14 replay log) *.imported (state found when a ledger was migrated) @@ -217,6 +218,7 @@ def ok(self) -> bool: "record.sealed": "sealed", "record.failed": "failed", "record.invalidated": "invalidated", + "record.reverified": "sealed", # an operator restored missing evidence (ledger.reverify) } diff --git a/stele/ledger/migration.py b/stele/ledger/migration.py index 487be65..64c760e 100644 --- a/stele/ledger/migration.py +++ b/stele/ledger/migration.py @@ -23,7 +23,10 @@ archive and verified against the recorded manifest (from disk, or from the archive if it already holds the bundle). A committed record whose bundle cannot be verified becomes INVALIDATED with the reason: the ledger can no - longer vouch for that content. + longer vouch for that content. The reason says whether the evidence was + missing (neither on disk nor archived) or mismatched; only a record with + missing evidence can be restored later, by ledger.reverify, from files that + still hash as recorded. - source_hash is kept only when it is the digest of exactly one Snapshot in the archive. Otherwise it moves to legacy_source_hash, which is unverified audit data and never used as provenance. @@ -55,6 +58,11 @@ ) +# Why a committed record was invalidated by migration (prefix of its error). +EVIDENCE_MISSING = "evidence missing" +EVIDENCE_MISMATCHED = "evidence mismatched" + + class LedgerMigrationError(Exception): """Raised when an existing ledger cannot be migrated; nothing is changed.""" @@ -162,9 +170,13 @@ def _migrate_row(old: dict, archive: BlobStore) -> dict: ) as exc: row["state"] = "invalidated" row["finalized_at"] = datetime.now(timezone.utc).isoformat() + # Missing evidence (neither on disk nor archived) can be restored + # by an operator who still has the files (ledger.reverify); + # anything else means the content is not what was recorded. + kind = EVIDENCE_MISSING if isinstance(exc, MissingArtifactError) else EVIDENCE_MISMATCHED row["error"] = ( - "INVALIDATED: migration — committed bundle could not be archived " - f"and verified: {exc!r}" + f"INVALIDATED: migration — {kind}: committed bundle could not be " + f"archived and verified: {exc!r}" ) else: row["state"] = "sealed" diff --git a/stele/ledger/reverify.py b/stele/ledger/reverify.py new file mode 100644 index 0000000..fa0c213 --- /dev/null +++ b/stele/ledger/reverify.py @@ -0,0 +1,148 @@ +""" +Restoring a record the migration invalidated for missing evidence (#39). + +Migrating a pre-#12 ledger invalidates a committed record whose bundle can be +neither archived from disk nor found in the archive. Often the evidence was +never archived (the archive did not exist yet) and the files are still +somewhere: invalidating is safe, but it is permanent, so such a record could +never be sealed again. + +reverify_record() is the explicit operator step that restores it. It never +happens during migration, and it is narrow: + +- the record must be INVALIDATED *by the migration*, for *missing* evidence. + Its error says so, and its event history is exactly the migration's import + of it: an invalidation by an operator or a divergent replay has a + record.invalidated event (or another error) and stays invalidated, as does + a record whose evidence was found to be different (mismatched); +- it must have no deliveries, since invalidation withdraws delivered data; +- the files in the given directory are hashed against the recorded manifest + and archived exactly as sealing does. Any difference, or a missing file, + leaves the record INVALIDATED. + +On success the record is SEALED again, its error cleared, and a +`record.reverified` event is appended to the hash chain in the same +transaction; the earlier events stay, so the history reads: imported as +invalidated, reverified. + + python -m stele.ledger.reverify --artifact-dir DIR +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from ..archive import BlobStore +from .events import EventLog +from .migration import EVIDENCE_MISSING +from .models import ArtifactRecord, ArtifactState +from .store import ( + ArtifactDriftError, + InvalidStateTransitionError, + LedgerStore, + MissingArtifactError, + RecordNotFoundError, + archive_bundle, +) + +_MIGRATION_PREFIX = "INVALIDATED: migration — " + + +class ReverifyRefused(Exception): + """The record cannot be restored; nothing was changed.""" + + +def reverifiable_problem(ledger: LedgerStore, record: ArtifactRecord) -> str | None: + """Why the record may not be reverified; None if it may.""" + if record.state is not ArtifactState.INVALIDATED: + return f"the record is {record.state.value}, not invalidated" + error = record.error or "" + if not error.startswith(f"{_MIGRATION_PREFIX}{EVIDENCE_MISSING}:"): + return "it was not invalidated by the migration for missing evidence" + log = EventLog(ledger) + try: + events = log.for_subject(record.record_id) + finally: + log.close() + if [e.kind for e in events] != ["record.imported"] or events[0].body.get("state") != "invalidated": + return "its history is not just the migration's import, so it was invalidated for another reason" + deliveries = ledger._conn.execute( + "SELECT COUNT(*) FROM deliveries WHERE record_id=?", (record.record_id,) + ).fetchone()[0] + if deliveries: + return "it has deliveries, and invalidating withdrew the delivered data" + return None + + +def reverify_record( + ledger: LedgerStore, record_id: str, artifact_dir: Path | str | None = None +) -> ArtifactRecord: + """Seal an invalidated record again from files that hash as recorded. + + artifact_dir defaults to the directory the record names. Raises + ReverifyRefused (nothing changed) for a record that may not be restored + or whose files do not match. + """ + try: + record = ledger.get(record_id) + except RecordNotFoundError: + raise ReverifyRefused(f"no record {record_id}") from None + problem = reverifiable_problem(ledger, record) + if problem is not None: + raise ReverifyRefused(f"record {record_id} cannot be reverified: {problem}") + directory = Path(artifact_dir) if artifact_dir is not None else Path(record.artifact_dir) + try: + archive_bundle(ledger.archive, directory, record.artifact_manifest, record.artifact_hash) + except (MissingArtifactError, ArtifactDriftError) as exc: + raise ReverifyRefused( + f"record {record_id}: the files in {directory} are not the recorded bundle: {exc}" + ) from exc + try: + ledger._transition( + record_id, + from_states=(ArtifactState.INVALIDATED,), + action="reverify", + set_sql="state='sealed', finalized_at=?, error=NULL", + params=(_now_iso(),), + event=("record.reverified", { + "artifact_hash": record.artifact_hash, + "previous_error": record.error, + }), + ) + except InvalidStateTransitionError as exc: + raise ReverifyRefused(str(exc)) from exc + return ledger.get(record_id) + + +def _now_iso() -> str: + from datetime import datetime, timezone + + return datetime.now(timezone.utc).isoformat() + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + prog="python -m stele.ledger.reverify", + description="Seal a migration-invalidated record again from files that hash as recorded.", + ) + ap.add_argument("ledger", type=Path) + ap.add_argument("archive", type=Path) + ap.add_argument("record_id") + ap.add_argument("--artifact-dir", type=Path, default=None, + help="where the record's files are now (default: the directory it names)") + args = ap.parse_args(argv) + ledger = LedgerStore(args.ledger, BlobStore(args.archive)) + try: + record = reverify_record(ledger, args.record_id, args.artifact_dir) + except ReverifyRefused as exc: + print(str(exc), file=sys.stderr) + return 1 + finally: + ledger.close() + print(f"record {record.record_id} is sealed again (artifact_hash {record.artifact_hash})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_reverify.py b/tests/test_reverify.py new file mode 100644 index 0000000..3f3954e --- /dev/null +++ b/tests/test_reverify.py @@ -0,0 +1,263 @@ +""" +Restoring records the migration invalidated for missing evidence (#39). + + 1. The migration says *why* it invalidated a committed record: missing + evidence (neither on disk nor archived) or mismatched evidence. + 2. reverify_record() seals a missing-evidence record again from files that + hash as recorded, logs record.reverified, and the ledger still verifies. + 3. It refuses everything else and changes nothing: mismatched evidence, + files that differ, records invalidated by an operator or a replay, + records with deliveries, records that are not invalidated. + 4. The CLI does the same. +""" +from __future__ import annotations + +import json +import shutil +import sqlite3 +import uuid +from pathlib import Path + +import pytest + +from stele.ledger.events import EventLog, verify_ledger +from stele.ledger.hashing import build_manifest, sha256_manifest +from stele.ledger.models import ArtifactState, ParserIdentity +from stele.ledger.reverify import ReverifyRefused, main as cli, reverify_record +from tests.ledger_helpers import open_ledger +from tests.test_ledger_provenance import _V0_DDL + + +@pytest.fixture +def v0(tmp_path: Path): + """A pre-#12 ledger; add(...) returns (record_id, output directory).""" + db = tmp_path / "ledger.db" + conn = sqlite3.connect(db) + conn.executescript(_V0_DDL) + counter = iter(range(100)) + + def add(state: str, *, keep_files: bool = True, error: str | None = None, + content: str | None = None) -> tuple[str, Path]: + n = next(counter) + out = tmp_path / f"out-{n}" + out.mkdir() + (out / "r.json").write_text(content or f"output {n}") + (out / "s.txt").write_text(f"second file {n}") + manifest = build_manifest(out, [out / "r.json", out / "s.txt"]) + if not keep_files: + shutil.rmtree(out) + record_id = str(uuid.uuid4()) + conn.execute( + "INSERT INTO artifact_records VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (record_id, str(uuid.uuid4()), None, None, str(out), json.dumps(manifest), + sha256_manifest(manifest), state, f"2026-06-26T00:00:{n:02d}+00:00", + None if state == "pending" else "2026-06-26T01:00:00+00:00", error), + ) + conn.commit() + return record_id, out + + yield db, add + conn.close() + + +def _restore(out: Path, n: int) -> None: + """Put back the files the v0 fixture wrote for output n.""" + out.mkdir(exist_ok=True) + (out / "r.json").write_text(f"output {n}") + (out / "s.txt").write_text(f"second file {n}") + + +def _kinds(ledger, record_id: str) -> list[str]: + log = EventLog(ledger) + try: + return [e.kind for e in log.for_subject(record_id)] + finally: + log.close() + + +# --------------------------------------------------------------------------- +# 1 — the migration says why +# --------------------------------------------------------------------------- + +def test_migration_distinguishes_missing_from_mismatched_evidence(v0, tmp_path) -> None: + db, add = v0 + missing, _ = add("committed", keep_files=False) + mismatched, out = add("committed") + (out / "r.json").write_text("not what was recorded") + + ledger = open_ledger(db) + assert ledger.get(missing).error.startswith("INVALIDATED: migration — evidence missing:") + assert ledger.get(mismatched).error.startswith("INVALIDATED: migration — evidence mismatched:") + + +# --------------------------------------------------------------------------- +# 2 — restoring +# --------------------------------------------------------------------------- + +def test_files_that_hash_as_recorded_seal_the_record_again(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + before = ledger.get(record_id) + assert before.state is ArtifactState.INVALIDATED + + _restore(out, 0) + record = reverify_record(ledger, record_id) + + assert record.state is ArtifactState.SEALED + assert record.error is None + assert record.artifact_hash == before.artifact_hash + assert ledger.archive.read_tree(record.artifact_hash) == record.artifact_manifest + assert _kinds(ledger, record_id) == ["record.imported", "record.reverified"] + report = verify_ledger(ledger) + assert report.ok, (report.chain.problems, report.problems) + + +def test_files_can_be_given_from_another_directory(v0, tmp_path) -> None: + db, add = v0 + record_id, _ = add("committed", keep_files=False) + elsewhere = tmp_path / "recovered" + _restore(elsewhere, 0) + ledger = open_ledger(db) + + assert reverify_record(ledger, record_id, elsewhere).state is ArtifactState.SEALED + assert verify_ledger(ledger).ok + + +def test_a_restored_record_is_an_ordinary_sealed_record(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + _restore(out, 0) + reverify_record(ledger, record_id) + shutil.rmtree(out) # the archive alone vouches for it now + reopened = open_ledger(db) + assert reopened.get(record_id).state is ArtifactState.SEALED + assert verify_ledger(reopened).ok + with pytest.raises(ReverifyRefused, match="not invalidated"): + reverify_record(reopened, record_id) + + +# --------------------------------------------------------------------------- +# 3 — everything else is refused, and nothing changes +# --------------------------------------------------------------------------- + +def _assert_untouched(ledger, record_id: str, events: list[str]) -> None: + assert ledger.get(record_id).state is ArtifactState.INVALIDATED + assert _kinds(ledger, record_id) == events + assert verify_ledger(ledger).ok + + +def test_mismatched_evidence_is_never_restored(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed") + (out / "r.json").write_text("not what was recorded") + ledger = open_ledger(db) + _restore(out, 0) # even with the right files back + with pytest.raises(ReverifyRefused, match="not invalidated by the migration for missing"): + reverify_record(ledger, record_id) + _assert_untouched(ledger, record_id, ["record.imported"]) + + +@pytest.mark.parametrize("damage", ["changed", "missing_file", "extra_only"]) +def test_files_that_differ_leave_the_record_invalidated(v0, tmp_path, damage) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + _restore(out, 0) + if damage == "changed": + (out / "s.txt").write_text("someone edited this") + elif damage == "missing_file": + (out / "s.txt").unlink() + else: + (out / "r.json").unlink() + (out / "s.txt").unlink() + (out / "other.txt").write_text("not in the manifest") + with pytest.raises(ReverifyRefused, match="not the recorded bundle"): + reverify_record(ledger, record_id) + _assert_untouched(ledger, record_id, ["record.imported"]) + + +def test_a_record_invalidated_by_an_operator_stays_invalidated(v0, tmp_path) -> None: + """Even with the migration's own words as the note, the history gives it away.""" + db, add = v0 + ledger = open_ledger(db) + out = tmp_path / "fresh" + out.mkdir() + (out / "a.txt").write_text("a") + pending = ledger.create_pending( + run_id=str(uuid.uuid4()), artifact_dir=out, artifact_paths=[out / "a.txt"], + parser=ParserIdentity("p", "1"), parser_config={}, + ) + ledger.seal(pending.record_id) + ledger.invalidate(pending.record_id, "migration — evidence missing: by hand") + with pytest.raises(ReverifyRefused, match="invalidated for another reason"): + reverify_record(ledger, pending.record_id, out) + assert ledger.get(pending.record_id).state is ArtifactState.INVALIDATED + + +def test_the_pre_reverify_error_text_is_not_trusted(v0, tmp_path) -> None: + """Invalidated by an earlier migration that did not say why: stays put.""" + db, add = v0 + record_id, out = add( + "invalidated", + error="INVALIDATED: migration — committed bundle could not be archived and verified: " + "MissingArtifactError('x')", + ) + ledger = open_ledger(db) + _restore(out, 0) + with pytest.raises(ReverifyRefused, match="not invalidated by the migration for missing"): + reverify_record(ledger, record_id) + _assert_untouched(ledger, record_id, ["record.imported"]) + + +def test_a_record_with_deliveries_is_refused(v0, tmp_path) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + conn = sqlite3.connect(db) + conn.execute( + "INSERT INTO deliveries (dispatch_id, record_id, target_kind, target, created_at) " + "VALUES ('d1', ?, 'T', '{}', '2026-06-26T02:00:00+00:00')", (record_id,), + ) + conn.commit() + conn.close() + _restore(out, 0) + with pytest.raises(ReverifyRefused, match="has deliveries"): + reverify_record(ledger, record_id) + assert ledger.get(record_id).state is ArtifactState.INVALIDATED + + +@pytest.mark.parametrize("state", ["committed", "pending", "failed"]) +def test_only_invalidated_records_are_reverified(v0, tmp_path, state) -> None: + db, add = v0 + record_id, _ = add(state, error="x" if state == "failed" else None) + ledger = open_ledger(db) + with pytest.raises(ReverifyRefused, match="not invalidated"): + reverify_record(ledger, record_id) + + +def test_unknown_record(v0) -> None: + ledger = open_ledger(v0[0]) + with pytest.raises(ReverifyRefused, match="no record"): + reverify_record(ledger, str(uuid.uuid4())) + + +# --------------------------------------------------------------------------- +# 4 — CLI +# --------------------------------------------------------------------------- + +def test_cli(v0, tmp_path, capsys) -> None: + db, add = v0 + record_id, out = add("committed", keep_files=False) + ledger = open_ledger(db) + archive = str(ledger.archive.root) + ledger.close() + + assert cli([str(db), archive, record_id]) == 1 # files are still gone + assert "not the recorded bundle" in capsys.readouterr().err + _restore(out, 0) + assert cli([str(db), archive, record_id, "--artifact-dir", str(out)]) == 0 + assert "sealed again" in capsys.readouterr().out + assert cli([str(db), archive, record_id]) == 1 # already sealed + assert "not invalidated" in capsys.readouterr().err