From e0010c19673a674b0f5caf1eed6ffbd7ccee4cda Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 16 Jul 2026 22:24:02 +0200 Subject: [PATCH 01/13] feat(storage): persist raw authority replay plans Problem: bounded raw replay selected components before parser census and reconstructed fairness from disposable daemon events. Plans could widen after selection, unselected work had no conserved outcome, and stale preconditions had no durable fail-closed blocker. What changed: add the source-v13 immutable census/plan/outcome ledger, split source census from index application, derive plan identity from logical and authority witnesses, record exact application receipts, require two matching quiescent censuses for fixed point, and project census handles through daemon/readiness surfaces. Verification: devtools test -k raw_materialization (81 passed); focused source/daemon/readiness selection (76 passed); devtools test -k raw_authority (6 passed); devtools verify --quick (16/16). Ref polylogue-hjpx.1. Ref polylogue-lkrc. Co-Authored-By: Claude --- docs/plans/topology-target.yaml | 24 +- docs/topology-status.md | 7 +- polylogue/daemon/cli.py | 34 +- polylogue/sources/revision_backfill.py | 241 +++++--- polylogue/storage/archive_readiness.py | 40 +- polylogue/storage/raw_authority.py | 547 ++++++++++++++++++ polylogue/storage/repair.py | 304 ++++++---- .../storage/sqlite/archive_tiers/source.py | 75 ++- .../source/013_raw_authority_ledger.sql | 71 +++ tests/unit/daemon/test_daemon_cli.py | 47 ++ tests/unit/storage/test_durable_migrations.py | 25 +- .../unit/storage/test_raw_authority_ledger.py | 165 ++++++ tests/unit/storage/test_repair.py | 358 +++++------- 13 files changed, 1504 insertions(+), 434 deletions(-) create mode 100644 polylogue/storage/raw_authority.py create mode 100644 polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql create mode 100644 tests/unit/storage/test_raw_authority_ledger.py diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index 693abe2ccc..8f2c8f103c 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -720,7 +720,7 @@ files: target: polylogue/browser_capture/actions.py owner: stable - path: polylogue/browser_capture/capture_jobs.py - loc: 366 + loc: 594 target: polylogue/browser_capture/capture_jobs.py owner: stable - path: polylogue/browser_capture/identity.py @@ -736,11 +736,11 @@ files: target: polylogue/browser_capture/receiver.py owner: stable - path: polylogue/browser_capture/route_contracts.py - loc: 203 + loc: 279 target: polylogue/browser_capture/route_contracts.py owner: stable - path: polylogue/browser_capture/server.py - loc: 693 + loc: 723 target: polylogue/browser_capture/server.py owner: stable - path: polylogue/cli/__init__.py @@ -1413,7 +1413,7 @@ files: target: polylogue/daemon/catchup_status.py owner: stable - path: polylogue/daemon/cli.py - loc: 1956 + loc: 1974 target: polylogue/daemon/cli.py owner: stable - path: polylogue/daemon/compare.py @@ -3162,7 +3162,7 @@ files: owner: stable cross_cut: { lifecycle: model } - path: polylogue/sources/revision_backfill.py - loc: 337 + loc: 407 target: polylogue/sources/revision_backfill.py owner: stable - path: polylogue/sources/source_acquisition.py @@ -3203,7 +3203,7 @@ files: target: TBD owner: storage-domain - path: polylogue/storage/archive_readiness.py - loc: 686 + loc: 722 target: TBD owner: storage-domain - path: polylogue/storage/archive_views.py @@ -3409,7 +3409,7 @@ files: target: polylogue/storage/insights/session/records.py owner: stable - path: polylogue/storage/insights/session/refresh.py - loc: 613 + loc: 688 target: polylogue/storage/insights/session/refresh.py owner: stable - path: polylogue/storage/insights/session/repair_assessment.py @@ -3483,13 +3483,17 @@ files: loc: 137 target: polylogue/storage/raw/models.py owner: stable + - path: polylogue/storage/raw_authority.py + loc: 542 + target: TBD + owner: storage-domain - path: polylogue/storage/raw_retention.py loc: 1317 target: polylogue/storage/raw_retention.py owner: storage-root reason: storage-root cross-cutting helper - path: polylogue/storage/repair.py - loc: 6909 + loc: 7006 target: polylogue/storage/repair.py owner: storage-root reason: storage-root cross-cutting helper @@ -3697,7 +3701,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/__init__.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive.py - loc: 11407 + loc: 11412 target: polylogue/storage/sqlite/archive_tiers/archive.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive_init.py @@ -3769,7 +3773,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/session_annotations_write.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/source.py - loc: 323 + loc: 396 target: polylogue/storage/sqlite/archive_tiers/source.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/source_write.py diff --git a/docs/topology-status.md b/docs/topology-status.md index 093c61e55d..946e4d6d93 100644 --- a/docs/topology-status.md +++ b/docs/topology-status.md @@ -31,9 +31,9 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe - **Stable** (no move scoped): 834 - **Kernel** (polylogue/ root): 7 - **Primitives** (storage-root): 18 -- **TBD** (cell needs explicit assignment): 7 -- **Total declared**: 995 -- **Realized polylogue/**/*.py**: 995 files declared +- **TBD** (cell needs explicit assignment): 8 +- **Total declared**: 996 +- **Realized polylogue/**/*.py**: 996 files declared ### TBD cells (require explicit routing) @@ -45,5 +45,6 @@ These rows in the projection have no resolved target yet. Each needs an explicit - `polylogue/storage/blob_publication.py` — no rule yet - `polylogue/storage/block_anchor.py` — no rule yet - `polylogue/storage/index_generation.py` — no rule yet +- `polylogue/storage/raw_authority.py` — no rule yet - `polylogue/storage/table_existence.py` — no rule yet diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 49dc71195f..899d89107f 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -642,16 +642,34 @@ def _emit_raw_materialization_pass(result: Any) -> None: from polylogue.daemon.events import emit_daemon_event metrics = dict(getattr(result, "metrics", {})) + census = getattr(result, "census_receipt", None) + census_payload = None + if census is not None: + census_payload = { + "census_id": census.census_id, + "sequence_no": census.sequence_no, + "inventory_digest": census.inventory_digest, + "residual_digest": census.residual_digest, + "plan_count": census.plan_count, + "executable_plan_count": census.executable_plan_count, + "residual_plan_count": census.residual_plan_count, + "predecessor_census_id": census.predecessor_census_id, + "fixed_point": census.fixed_point, + "query_handle": census.query_handle, + } + payload = { + "pass_id": f"raw-materialization:{os.urandom(16).hex()}", + "success": bool(result.success), + "repaired_count": int(result.repaired_count), + "detail": str(result.detail), + "metrics": metrics, + "plan_outcomes": [outcome.to_dict() for outcome in outcomes], + } + if census_payload is not None: + payload["census"] = census_payload emit_daemon_event( "raw_materialization_pass", - payload={ - "pass_id": f"raw-materialization:{os.urandom(16).hex()}", - "success": bool(result.success), - "repaired_count": int(result.repaired_count), - "detail": str(result.detail), - "metrics": metrics, - "plan_outcomes": [outcome.to_dict() for outcome in outcomes], - }, + payload=payload, ) diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 2b445e30de..6da31989b5 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -36,6 +36,25 @@ class RevisionBackfillResult: adoption_deferred: int = 0 +@dataclass(frozen=True, slots=True) +class RevisionCensusResult: + scanned: int + classified_full: int + quarantined: int + input_raw_ids: tuple[str, ...] + logical_keys: tuple[str, ...] + + +@dataclass(slots=True) +class _RevisionCensusState: + scanned: int + classified: int + quarantined: int + censused: set[str] + membership_candidates: dict[str, set[str]] + provisional_full_raw_ids: dict[str, set[str]] + + class RawRevisionReplayResourceBlockedError(RuntimeError): def __init__(self, raw_ids: list[str], limit_bytes: int, total_bytes: int) -> None: self.raw_ids = tuple(raw_ids) @@ -44,6 +63,121 @@ def __init__(self, raw_ids: list[str], limit_bytes: int, total_bytes: int) -> No super().__init__(f"{len(raw_ids)} raw revision(s) total {total_bytes} bytes exceed replay limit {limit_bytes}") +def _census_historical_revision_evidence( + archive: ArchiveStore, + spill: _ParsedSessionSpill, + *, + selected_raw_ids: list[str] | None, + max_payload_bytes: int | None, +) -> _RevisionCensusState: + """Persist a complete bounded parser census without mutating index.db.""" + state = _RevisionCensusState(0, 0, 0, set(), {}, {}) + census_selections: tuple[tuple[str, ...] | None, ...] + if selected_raw_ids is None: + census_selections = (None,) + else: + census_selections = archive.raw_membership_selection_components(selected_raw_ids) + for initial_selection in census_selections: + census_selection = initial_selection + while True: + rows = archive.raw_membership_census_rows(census_selection) + pending_rows = [(raw_id, source_index) for raw_id, source_index in rows if raw_id not in state.censused] + if max_payload_bytes is not None: + payload_sizes = archive.raw_payload_sizes([raw_id for raw_id, _index in rows]) + total_payload_bytes = sum(payload_sizes.values()) + oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] + if oversized or total_payload_bytes > max_payload_bytes: + blocked_ids = oversized or list(payload_sizes) + raise RawRevisionReplayResourceBlockedError( + sorted(blocked_ids), max_payload_bytes, total_payload_bytes + ) + for raw_id, source_index in pending_rows: + state.scanned += 1 + state.censused.add(raw_id) + if source_index < 0: + archive.replace_raw_membership_census( + raw_id, + None, + parser_fingerprint="revision-membership-v1", + censused_at_ms=0, + detail=BYTE_AUTHORITY_CENSUS_DETAIL, + ) + state.quarantined += 1 + continue + try: + sessions, payload_bytes, revision_kind = _parse_retained_raw(archive, raw_id) + except Exception as exc: + archive.replace_raw_membership_census( + raw_id, + None, + parser_fingerprint="revision-membership-v1", + censused_at_ms=0, + detail=str(exc), + ) + state.quarantined += 1 + continue + state.classified += int(len(sessions) == 1) + spill.add(raw_id, sessions, payload_bytes=payload_bytes) + if len(sessions) == 1 and revision_kind is RawRevisionKind.UNKNOWN: + session = sessions[0] + logical_key = f"{session.source_name.value}:{session.provider_session_id}" + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key=logical_key, + kind=RawRevisionKind.FULL, + source_revision=raw_id, + acquisition_generation=0, + authority=RawRevisionAuthority.QUARANTINED, + ), + ) + state.provisional_full_raw_ids.setdefault(logical_key, set()).add(raw_id) + elif revision_kind is RawRevisionKind.UNKNOWN: + archive.replace_raw_membership_census( + raw_id, + sessions, + parser_fingerprint="revision-membership-v1", + censused_at_ms=0, + ) + for session in sessions: + logical_key = f"{session.source_name.value}:{session.provider_session_id}" + state.membership_candidates.setdefault(logical_key, set()).add(raw_id) + if census_selection is None: + break + expanded, _keys = archive.expand_raw_membership_selection(list(census_selection)) + if set(expanded) == set(census_selection): + break + census_selection = expanded + return state + + +def census_historical_revision_evidence( + archive_root: Path, + *, + selected_raw_ids: list[str] | None = None, + max_payload_bytes: int | None = None, +) -> RevisionCensusResult: + """Complete the source-tier census stage without applying index changes.""" + with ( + ArchiveStore.open_existing(archive_root, read_only=False) as archive, + _ParsedSessionSpill(archive_root) as spill, + ): + state = _census_historical_revision_evidence( + archive, + spill, + selected_raw_ids=selected_raw_ids, + max_payload_bytes=max_payload_bytes, + ) + expanded, logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) + return RevisionCensusResult( + state.scanned, + state.classified, + state.quarantined, + expanded, + logical_keys, + ) + + def backfill_historical_revision_evidence( archive_root: Path, *, @@ -58,13 +192,9 @@ def backfill_historical_revision_evidence( loaded one logical authority cohort at a time. Peak retained session trees therefore follow the largest raw/cohort, not the archive-wide raw count. """ - scanned = 0 - classified = 0 - quarantined = 0 adoption_deferred = 0 + quarantined = 0 logical_keys: set[str] = set() - membership_candidates: dict[str, set[str]] = {} - provisional_full_raw_ids: dict[str, set[str]] = {} archive_context = ( ArchiveStore.open_owned_inactive_generation( archive_root, @@ -75,83 +205,14 @@ def backfill_historical_revision_evidence( else ArchiveStore.open_existing(archive_root, read_only=False) ) with archive_context as archive, _ParsedSessionSpill(archive_root) as spill: - census_selections: tuple[tuple[str, ...] | None, ...] - if selected_raw_ids is None: - census_selections = (None,) - else: - census_selections = archive.raw_membership_selection_components(selected_raw_ids) - censused: set[str] = set() - for initial_selection in census_selections: - census_selection = initial_selection - while True: - rows = archive.raw_membership_census_rows(census_selection) - pending_rows = [(raw_id, source_index) for raw_id, source_index in rows if raw_id not in censused] - if max_payload_bytes is not None: - payload_sizes = archive.raw_payload_sizes([raw_id for raw_id, _index in rows]) - total_payload_bytes = sum(payload_sizes.values()) - oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] - if oversized or total_payload_bytes > max_payload_bytes: - blocked_ids = oversized or list(payload_sizes) - raise RawRevisionReplayResourceBlockedError( - sorted(blocked_ids), max_payload_bytes, total_payload_bytes - ) - for raw_id, source_index in pending_rows: - scanned += 1 - censused.add(raw_id) - if source_index < 0: - archive.replace_raw_membership_census( - raw_id, - None, - parser_fingerprint="revision-membership-v1", - censused_at_ms=0, - detail=BYTE_AUTHORITY_CENSUS_DETAIL, - ) - quarantined += 1 - continue - try: - sessions, _payload_bytes, revision_kind = _parse_retained_raw(archive, raw_id) - except Exception as exc: - archive.replace_raw_membership_census( - raw_id, - None, - parser_fingerprint="revision-membership-v1", - censused_at_ms=0, - detail=str(exc), - ) - quarantined += 1 - continue - classified += int(len(sessions) == 1) - spill.add(raw_id, sessions, payload_bytes=_payload_bytes) - if len(sessions) == 1 and revision_kind is RawRevisionKind.UNKNOWN: - session = sessions[0] - logical_key = f"{session.source_name.value}:{session.provider_session_id}" - archive.bind_raw_revision( - raw_id, - RawRevisionEnvelope( - logical_source_key=logical_key, - kind=RawRevisionKind.FULL, - source_revision=raw_id, - acquisition_generation=0, - authority=RawRevisionAuthority.QUARANTINED, - ), - ) - provisional_full_raw_ids.setdefault(logical_key, set()).add(raw_id) - elif revision_kind is RawRevisionKind.UNKNOWN: - archive.replace_raw_membership_census( - raw_id, - sessions, - parser_fingerprint="revision-membership-v1", - censused_at_ms=0, - ) - for session in sessions: - logical_key = f"{session.source_name.value}:{session.provider_session_id}" - membership_candidates.setdefault(logical_key, set()).add(raw_id) - if census_selection is None: - break - expanded, _keys = archive.expand_raw_membership_selection(list(census_selection)) - if set(expanded) == set(census_selection): - break - census_selection = expanded + census = _census_historical_revision_evidence( + archive, + spill, + selected_raw_ids=selected_raw_ids, + max_payload_bytes=max_payload_bytes, + ) + membership_candidates = census.membership_candidates + provisional_full_raw_ids = census.provisional_full_raw_ids _unclassified, selected_keys = archive.raw_revision_rebuild_selection(selected_raw_ids) logical_keys.update(selected_keys) @@ -245,7 +306,13 @@ def backfill_historical_revision_evidence( ) if classification.accepted_raw_ids: replayed += 1 - return RevisionBackfillResult(scanned, classified, replayed, quarantined, adoption_deferred) + return RevisionBackfillResult( + census.scanned, + census.classified, + replayed, + census.quarantined + quarantined, + adoption_deferred, + ) def _parse_retained_raw(archive: ArchiveStore, raw_id: str) -> tuple[list[ParsedSession], int, RawRevisionKind]: @@ -334,4 +401,10 @@ def _parse_one(provider: Provider, payload: bytes, source_path: str) -> list[Par ) -__all__ = ["RawRevisionReplayResourceBlockedError", "RevisionBackfillResult", "backfill_historical_revision_evidence"] +__all__ = [ + "RawRevisionReplayResourceBlockedError", + "RevisionBackfillResult", + "RevisionCensusResult", + "backfill_historical_revision_evidence", + "census_historical_revision_evidence", +] diff --git a/polylogue/storage/archive_readiness.py b/polylogue/storage/archive_readiness.py index 9a730fd59d..77d722caff 100644 --- a/polylogue/storage/archive_readiness.py +++ b/polylogue/storage/archive_readiness.py @@ -228,6 +228,38 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob ) lost_source_evidence_count = _missing_source_raw_session_count(conn) lost_source_evidence_samples = _missing_source_raw_session_samples(conn) + authority_census: dict[str, object] | None = None + if _table_columns(conn, "source", "raw_authority_censuses"): + census_row = conn.execute( + """ + SELECT census_id, sequence_no, inventory_digest, residual_digest, + plan_count, executable_plan_count, residual_plan_count, + predecessor_census_id, fixed_point, completed_at_ms + FROM source.raw_authority_censuses + ORDER BY sequence_no DESC LIMIT 1 + """ + ).fetchone() + if census_row is not None: + authority_census = { + "census_id": str(census_row["census_id"]), + "sequence_no": int(census_row["sequence_no"]), + "inventory_digest": str(census_row["inventory_digest"]), + "residual_digest": str(census_row["residual_digest"]), + "plan_count": int(census_row["plan_count"]), + "executable_plan_count": int(census_row["executable_plan_count"]), + "residual_plan_count": int(census_row["residual_plan_count"]), + "predecessor_census_id": census_row["predecessor_census_id"], + "fixed_point": bool(census_row["fixed_point"]), + "completed_at_ms": int(census_row["completed_at_ms"]), + "query_handle": f"raw-authority-census:{census_row['census_id']}", + } + authority_blocker_count = 0 + if _table_columns(conn, "source", "raw_authority_blockers"): + authority_blocker_count = int( + conn.execute( + "SELECT COUNT(*) FROM source.raw_authority_blockers WHERE resolved_at_ms IS NULL" + ).fetchone()[0] + ) except Exception as exc: return { "available": False, @@ -258,6 +290,8 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob } if adoption_deferred_count: category_counts["adoption_deferred"] = adoption_deferred_count + if authority_blocker_count: + category_counts["raw_authority_blocker"] = authority_blocker_count category_counts.update( {category: count for category, count in classified_counts.items() if category != "parse-failed"} ) @@ -273,12 +307,12 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "critical": critical, "warning": 0, "actionable": actionable, - "blocked": adoption_deferred_count, + "blocked": adoption_deferred_count + authority_blocker_count, "classified": classified, "unchecked": unchecked, "affected_total": total, "affected_actionable": affected_actionable, - "affected_blocked": adoption_deferred_count, + "affected_blocked": adoption_deferred_count + authority_blocker_count, "affected_open": 0, "affected_classified": classified, "affected_unchecked": unchecked, @@ -286,6 +320,8 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "lost_source_evidence_samples": lost_source_evidence_samples, "category_counts": category_counts, "source_family_counts": {str(item["origin"]): int(item["count"] or 0) for item in family_rows}, + "raw_authority_census": authority_census, + "raw_authority_blocker_count": authority_blocker_count, } diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py new file mode 100644 index 0000000000..6105d9906a --- /dev/null +++ b/polylogue/storage/raw_authority.py @@ -0,0 +1,547 @@ +"""Durable immutable plans and conservation receipts for raw reconciliation. + +The source tier is the authority for this ledger. ``index.db`` may be rebuilt +and ``ops.db`` may be deleted; neither event is allowed to erase replay +fairness, stale-plan blockers, or the proof that a complete census was +accounted for. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import time +from collections.abc import Mapping, Sequence +from contextlib import closing +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path + +from polylogue.core.json import JSONDocument, json_document +from polylogue.logging import get_logger + +RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v1" +logger = get_logger(__name__) + + +class RawReplayPlanStatus(StrEnum): + EXECUTED = "executed" + RETRYABLE = "retryable" + DEFERRED = "deferred" + TERMINAL = "terminal" + REJECTED_STALE = "rejected_stale" + CARRIED_FORWARD = "carried_forward" + + +@dataclass(frozen=True, slots=True) +class RawReplayPlan: + plan_id: str + input_digest: str + input_raw_ids: tuple[str, ...] + logical_keys: tuple[str, ...] + authority_witness: JSONDocument + source_preconditions: JSONDocument + index_preconditions: JSONDocument + + def to_dict(self) -> JSONDocument: + return json_document( + { + "plan_id": self.plan_id, + "input_digest": self.input_digest, + "input_raw_ids": list(self.input_raw_ids), + "logical_keys": list(self.logical_keys), + "authority_witness": self.authority_witness, + "source_preconditions": self.source_preconditions, + "index_preconditions": self.index_preconditions, + } + ) + + +@dataclass(frozen=True, slots=True) +class RawReplayPlanOutcome: + plan_id: str + input_raw_ids: tuple[str, ...] + status: RawReplayPlanStatus + reason: str + next_action: str + application_receipt: JSONDocument | None = None + + def to_dict(self) -> JSONDocument: + payload: dict[str, object] = { + "plan_id": self.plan_id, + "input_raw_ids": list(self.input_raw_ids), + "status": self.status.value, + "reason": self.reason, + "next_action": self.next_action, + } + if self.application_receipt is not None: + payload["application_receipt"] = self.application_receipt + return json_document(payload) + + +@dataclass(frozen=True, slots=True) +class RawAuthorityCensusReceipt: + census_id: str + sequence_no: int + inventory_digest: str + residual_digest: str + plan_count: int + executable_plan_count: int + residual_plan_count: int + predecessor_census_id: str | None + fixed_point: bool + + @property + def query_handle(self) -> str: + return f"raw-authority-census:{self.census_id}" + + +def _canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +def _json_value(value: object) -> object: + if isinstance(value, bytes): + return value.hex() + return value + + +def _rows(conn: sqlite3.Connection, sql: str, params: Sequence[object] = ()) -> list[dict[str, object]]: + cursor = conn.execute(sql, tuple(params)) + names = tuple(column[0] for column in cursor.description or ()) + return [{name: _json_value(value) for name, value in zip(names, row, strict=True)} for row in cursor] + + +def build_raw_replay_plan(conn: sqlite3.Connection, input_raw_ids: Sequence[str]) -> RawReplayPlan: + """Snapshot one complete component from an attached source/index pair.""" + raw_ids = tuple(sorted(dict.fromkeys(input_raw_ids))) + if not raw_ids: + raise ValueError("raw replay plan requires at least one input raw id") + marks = ",".join("?" for _ in raw_ids) + source_rows = _rows( + conn, + f""" + SELECT raw_id, origin, native_id, source_path, source_index, + hex(blob_hash) AS blob_hash, blob_size, logical_source_key, + revision_kind, source_revision, predecessor_source_revision, + predecessor_raw_id, baseline_raw_id, append_start_offset, + append_end_offset, acquisition_generation, revision_authority + FROM raw_sessions WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + raw_ids, + ) + if tuple(str(row["raw_id"]) for row in source_rows) != raw_ids: + raise RuntimeError("raw replay plan input disappeared during census") + membership_rows = _rows( + conn, + f""" + SELECT raw_id, logical_source_key, provider_session_id, source_revision, + hex(normalized_content_hash) AS normalized_content_hash, + message_count, predecessor_raw_id, acquisition_generation, + revision_authority, decision + FROM raw_session_memberships + WHERE raw_id IN ({marks}) + ORDER BY raw_id, logical_source_key + """, + raw_ids, + ) + census_rows = _rows( + conn, + f""" + SELECT raw_id, parser_fingerprint, status, member_count, detail + FROM raw_membership_census + WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + raw_ids, + ) + logical_keys = tuple( + sorted( + { + str(value) + for row in (*source_rows, *membership_rows) + if (value := row.get("logical_source_key")) is not None + } + ) + ) + if logical_keys: + key_marks = ",".join("?" for _ in logical_keys) + head_rows = _rows( + conn, + f""" + SELECT logical_source_key, session_id, accepted_raw_id, + accepted_source_revision, hex(accepted_content_hash) AS accepted_content_hash, + accepted_frontier_kind, accepted_frontier, + acquisition_generation, append_end_offset + FROM index_tier.raw_revision_heads + WHERE logical_source_key IN ({key_marks}) ORDER BY logical_source_key + """, + logical_keys, + ) + else: + head_rows = [] + session_rows = _rows( + conn, + f""" + SELECT session_id, raw_id, hex(content_hash) AS content_hash, message_count + FROM index_tier.sessions + WHERE raw_id IN ({marks}) ORDER BY session_id + """, + raw_ids, + ) + authority_witness = json_document( + { + "membership_census": census_rows, + "memberships": membership_rows, + "revision_heads": head_rows, + } + ) + source_preconditions = json_document({"raw_sessions": source_rows}) + index_preconditions = json_document({"sessions": session_rows, "revision_heads": head_rows}) + identity = { + "schema": "polylogue.raw-replay-plan.v2", + "input_raw_ids": list(raw_ids), + "logical_keys": list(logical_keys), + "authority_witness": authority_witness, + "source_preconditions": source_preconditions, + "index_preconditions": index_preconditions, + } + input_digest = _digest(identity) + return RawReplayPlan( + plan_id=f"raw-replay:{input_digest}", + input_digest=input_digest, + input_raw_ids=raw_ids, + logical_keys=logical_keys, + authority_witness=authority_witness, + source_preconditions=source_preconditions, + index_preconditions=index_preconditions, + ) + + +def build_raw_replay_plans(archive_root: Path, components: Sequence[tuple[str, ...]]) -> tuple[RawReplayPlan, ...]: + if not components: + return () + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: + conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + return tuple(build_raw_replay_plan(conn, component) for component in components) + + +def raw_replay_plan_last_attempts(archive_root: Path) -> dict[str, int]: + """Return durable attempt order; deleting ops.db cannot reset fairness.""" + source_db = archive_root / "source.db" + if not source_db.is_file(): + return {} + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='raw_authority_census_plans'" + ).fetchone() + if exists is None: + return {} + return { + str(row[0]): int(row[1]) + for row in conn.execute( + """ + SELECT plan_id, MAX(recorded_at_ms) + FROM raw_authority_census_plans + WHERE selected = 1 GROUP BY plan_id + """ + ) + } + + +def unresolved_raw_authority_blockers(archive_root: Path) -> int: + source_db = archive_root / "source.db" + if not source_db.is_file(): + return 0 + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='raw_authority_blockers'" + ).fetchone() + if exists is None: + return 0 + return int( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] + ) + + +def record_raw_authority_census( + archive_root: Path, + plans: Sequence[RawReplayPlan], + *, + selected_plan_ids: set[str], + scope: Mapping[str, object], + residual: Mapping[str, object], +) -> RawAuthorityCensusReceipt: + """Atomically publish a complete plan census with carried-forward outcomes.""" + now = int(time.time() * 1000) + inventory_digest = _digest([plan.plan_id for plan in plans]) + residual_digest = _digest(residual) + scope_json = _canonical_json(scope) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + previous = conn.execute( + """ + SELECT census_id, sequence_no, inventory_digest, residual_digest, + executable_plan_count, scope_json + FROM raw_authority_censuses ORDER BY sequence_no DESC LIMIT 1 + """ + ).fetchone() + sequence_no = int(previous[1]) + 1 if previous is not None else 1 + predecessor = str(previous[0]) if previous is not None else None + executable_count = len(selected_plan_ids) + fixed_point = bool( + previous is not None + and int(previous[4]) == 0 + and executable_count == 0 + and str(previous[2]) == inventory_digest + and str(previous[3]) == residual_digest + and str(previous[5]) == scope_json + ) + census_id = f"census:{sequence_no}:{inventory_digest[:16]}:{residual_digest[:16]}" + for plan in plans: + values = ( + plan.plan_id, + plan.input_digest, + _canonical_json(list(plan.input_raw_ids)), + _canonical_json(list(plan.logical_keys)), + _canonical_json(plan.authority_witness), + _canonical_json(plan.source_preconditions), + _canonical_json(plan.index_preconditions), + now, + ) + conn.execute( + """ + INSERT INTO raw_authority_plans ( + plan_id, input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(plan_id) DO NOTHING + """, + values, + ) + stored = conn.execute( + """ + SELECT input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json + FROM raw_authority_plans WHERE plan_id = ? + """, + (plan.plan_id,), + ).fetchone() + if stored != values[1:7]: + raise RuntimeError(f"immutable raw replay plan collision: {plan.plan_id}") + conn.execute( + """ + INSERT INTO raw_authority_censuses ( + census_id, sequence_no, scope_json, parser_fingerprint, + inventory_digest, residual_digest, plan_count, + executable_plan_count, residual_plan_count, + predecessor_census_id, fixed_point, created_at_ms, completed_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + census_id, + sequence_no, + scope_json, + RAW_AUTHORITY_PARSER_FINGERPRINT, + inventory_digest, + residual_digest, + len(plans), + executable_count, + len(plans) - executable_count, + predecessor, + int(fixed_point), + now, + now, + ), + ) + for ordinal, plan in enumerate(plans): + selected = plan.plan_id in selected_plan_ids + conn.execute( + """ + INSERT INTO raw_authority_census_plans ( + census_id, plan_id, ordinal, selected, outcome_status, + reason, next_action, application_receipt_json, recorded_at_ms + ) VALUES (?, ?, ?, ?, 'carried_forward', ?, ?, '{}', ?) + """, + ( + census_id, + plan.plan_id, + ordinal, + int(selected), + "selected plan awaits a typed application outcome" + if selected + else "bounded scheduler carried this complete plan forward unchanged", + "execute this plan in the current pass" if selected else "retain for a later bounded pass", + now, + ), + ) + return RawAuthorityCensusReceipt( + census_id=census_id, + sequence_no=sequence_no, + inventory_digest=inventory_digest, + residual_digest=residual_digest, + plan_count=len(plans), + executable_plan_count=executable_count, + residual_plan_count=len(plans) - executable_count, + predecessor_census_id=predecessor, + fixed_point=fixed_point, + ) + + +def validate_raw_replay_plan(archive_root: Path, plan: RawReplayPlan) -> tuple[bool, JSONDocument]: + try: + observed = build_raw_replay_plans(archive_root, (plan.input_raw_ids,))[0] + except Exception as exc: + logger.warning("raw replay plan validation could not rebuild %s", plan.plan_id, exc_info=True) + return False, json_document({"error": f"{type(exc).__name__}: {exc}"}) + return observed == plan, observed.to_dict() + + +def raw_replay_application_receipt(archive_root: Path, plan: RawReplayPlan) -> JSONDocument: + marks = ",".join("?" for _ in plan.input_raw_ids) + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: + conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + source = _rows( + conn, + f""" + SELECT raw_id, parsed_at_ms, parse_error + FROM raw_sessions WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + plan.input_raw_ids, + ) + memberships = _rows( + conn, + f""" + SELECT raw_id, logical_source_key, decision, decided_at_ms + FROM raw_session_memberships + WHERE raw_id IN ({marks}) ORDER BY raw_id, logical_source_key + """, + plan.input_raw_ids, + ) + applications = _rows( + conn, + f""" + SELECT decision_id, raw_id, session_id, logical_source_key, decision, + accepted_raw_id, hex(accepted_content_hash) AS accepted_content_hash, + decided_at_ms + FROM index_tier.raw_revision_applications + WHERE raw_id IN ({marks}) ORDER BY raw_id, decision_id + """, + plan.input_raw_ids, + ) + return json_document( + { + "schema": "polylogue.raw-replay-application-receipt.v1", + "source_rows": source, + "membership_rows": memberships, + "application_rows": applications, + } + ) + + +def record_raw_replay_outcome( + archive_root: Path, + census_id: str, + outcome: RawReplayPlanOutcome, +) -> None: + now = int(time.time() * 1000) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + updated = conn.execute( + """ + UPDATE raw_authority_census_plans + SET outcome_status = ?, reason = ?, next_action = ?, + application_receipt_json = ?, recorded_at_ms = ? + WHERE census_id = ? AND plan_id = ? AND selected = 1 + """, + ( + outcome.status.value, + outcome.reason, + outcome.next_action, + _canonical_json(outcome.application_receipt or {}), + now, + census_id, + outcome.plan_id, + ), + ).rowcount + if updated != 1: + raise RuntimeError(f"outcome does not conserve one selected plan: {outcome.plan_id}") + + +def reject_stale_raw_replay_plan( + archive_root: Path, + census_id: str, + plan: RawReplayPlan, + observed: JSONDocument, +) -> RawReplayPlanOutcome: + """Persist the fail-closed blocker before returning observational output.""" + now = int(time.time() * 1000) + blocker_id = f"raw-authority-blocker:{_digest([plan.plan_id, observed])}" + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.REJECTED_STALE, + "immutable source/index preconditions changed after the census", + "resolve the durable raw-authority blocker before automatic convergence resumes", + json_document({"expected": plan.to_dict(), "observed": observed, "blocker_id": blocker_id}), + ) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + conn.execute( + """ + INSERT INTO raw_authority_blockers ( + blocker_id, plan_id, census_id, reason, expected_json, + observed_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(blocker_id) DO NOTHING + """, + ( + blocker_id, + plan.plan_id, + census_id, + outcome.reason, + _canonical_json(plan.to_dict()), + _canonical_json(observed), + now, + ), + ) + updated = conn.execute( + """ + UPDATE raw_authority_census_plans + SET outcome_status = 'rejected_stale', reason = ?, next_action = ?, + application_receipt_json = ?, recorded_at_ms = ? + WHERE census_id = ? AND plan_id = ? AND selected = 1 + """, + ( + outcome.reason, + outcome.next_action, + _canonical_json(outcome.application_receipt or {}), + now, + census_id, + plan.plan_id, + ), + ).rowcount + if updated != 1: + raise RuntimeError(f"stale rejection does not conserve one selected plan: {plan.plan_id}") + return outcome + + +__all__ = [ + "RAW_AUTHORITY_PARSER_FINGERPRINT", + "RawAuthorityCensusReceipt", + "RawReplayPlan", + "RawReplayPlanOutcome", + "RawReplayPlanStatus", + "build_raw_replay_plan", + "build_raw_replay_plans", + "raw_replay_application_receipt", + "raw_replay_plan_last_attempts", + "record_raw_authority_census", + "record_raw_replay_outcome", + "reject_stale_raw_replay_plan", + "unresolved_raw_authority_blockers", + "validate_raw_replay_plan", +] diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index d9391722e0..4ccccc01cb 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -14,7 +14,6 @@ from contextlib import closing, contextmanager, suppress from dataclasses import dataclass, field from datetime import UTC, datetime -from enum import StrEnum from pathlib import Path from typing import cast @@ -57,6 +56,20 @@ count_messages_by_type_sync, count_unclassified_message_type_sync, ) +from polylogue.storage.raw_authority import ( + RawAuthorityCensusReceipt, + RawReplayPlan, + RawReplayPlanOutcome, + RawReplayPlanStatus, + build_raw_replay_plans, + raw_replay_application_receipt, + raw_replay_plan_last_attempts, + record_raw_authority_census, + record_raw_replay_outcome, + reject_stale_raw_replay_plan, + unresolved_raw_authority_blockers, + validate_raw_replay_plan, +) logger = get_logger(__name__) _MAINTENANCE_TARGET_CATALOG = build_maintenance_target_catalog() @@ -4588,34 +4601,6 @@ def max_blob_bytes(self) -> int: return max((self.raw_blob_bytes.get(raw_id, 0) for raw_id in self.raw_ids), default=0) -class RawReplayPlanStatus(StrEnum): - EXECUTED = "executed" - RETRYABLE = "retryable" - DEFERRED = "deferred" - TERMINAL = "terminal" - REJECTED_STALE = "rejected_stale" - - -@dataclass(frozen=True, slots=True) -class RawReplayPlanOutcome: - plan_id: str - input_raw_ids: tuple[str, ...] - status: RawReplayPlanStatus - reason: str - next_action: str - - def to_dict(self) -> JSONDocument: - return json_document( - { - "plan_id": self.plan_id, - "input_raw_ids": list(self.input_raw_ids), - "status": self.status.value, - "reason": self.reason, - "next_action": self.next_action, - } - ) - - def _raw_materialization_origin_from_provider(provider: str | None) -> str | None: if provider is None: return None @@ -4896,7 +4881,9 @@ def _raw_materialization_ordered_components( candidate_ids = set(candidates.raw_ids) source_components = candidates.authority_components or tuple((raw_id,) for raw_id in candidates.raw_ids) components = [component for component in source_components if candidate_ids.intersection(component)] - last_attempts = _raw_replay_plan_last_attempts(archive_root) + plans = build_raw_replay_plans(archive_root, components) + plan_ids = {plan.input_raw_ids: plan.plan_id for plan in plans} + last_attempts = raw_replay_plan_last_attempts(archive_root) def candidate_order(raw_id: str) -> tuple[int, int, str]: if raw_id in candidates.raw_acquired_at_ms: @@ -4906,8 +4893,8 @@ def candidate_order(raw_id: str) -> tuple[int, int, str]: return sorted( components, key=lambda component: ( - _raw_replay_plan_id(component) in last_attempts, - last_attempts.get(_raw_replay_plan_id(component), 0), + plan_ids[component] in last_attempts, + last_attempts.get(plan_ids[component], 0), min(candidate_order(raw_id) for raw_id in component if raw_id in candidate_ids), component, ), @@ -4933,50 +4920,15 @@ def _raw_materialization_component_blob_bytes(candidates: RawMaterializationCand return candidates.expanded_blob_bytes.get(raw_id, candidates.raw_blob_bytes.get(raw_id, 0)) -def _raw_replay_plan_id(component: tuple[str, ...]) -> str: - digest = hashlib.sha256() - digest.update(b"polylogue.raw-replay-plan.v1\0") - for raw_id in sorted(component): - digest.update(raw_id.encode("utf-8")) - digest.update(b"\0") - return f"raw-replay:{digest.hexdigest()}" - - -def _raw_replay_plan_last_attempts(archive_root: Path) -> dict[str, int]: - """Read disposable daemon receipts to rotate retryable plans fairly.""" - ops_db = archive_root / "ops.db" - if not ops_db.is_file(): - return {} - try: - with closing(sqlite3.connect(f"file:{ops_db}?mode=ro", uri=True)) as conn: - exists = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='daemon_events'").fetchone() - if exists is None: - return {} - rows = conn.execute( - """ - SELECT json_extract(outcome.value, '$.plan_id'), MAX(event.id) - FROM daemon_events AS event, - json_each(event.payload_json, '$.plan_outcomes') AS outcome - WHERE event.kind = 'raw_materialization_pass' - AND json_valid(event.payload_json) - AND json_extract(outcome.value, '$.plan_id') IS NOT NULL - GROUP BY json_extract(outcome.value, '$.plan_id') - """ - ) - return {str(row[0]): int(row[1]) for row in rows} - except sqlite3.Error: - logger.warning("raw replay: failed to read prior plan attempts", exc_info=True) - return {} - - def _raw_replay_plan_outcome( conn: sqlite3.Connection, - component: tuple[str, ...], + plan: RawReplayPlan, *, remaining: RawMaterializationCandidates, ) -> RawReplayPlanOutcome: """Conserve one selected component into an explicit post-pass state.""" - plan_id = _raw_replay_plan_id(component) + plan_id = plan.plan_id + component = plan.input_raw_ids remaining_ids = set(remaining.expanded_raw_ids) | set(remaining.raw_ids) if remaining_ids.intersection(component): return RawReplayPlanOutcome( @@ -5067,16 +5019,16 @@ def _raw_replay_plan_outcome( def _raw_replay_plan_outcomes( archive_root: Path, - components: Sequence[tuple[str, ...]], + plans: Sequence[RawReplayPlan], *, remaining: RawMaterializationCandidates, ) -> tuple[RawReplayPlanOutcome, ...]: - if not components: + if not plans: return () with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: conn.row_factory = sqlite3.Row conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) - return tuple(_raw_replay_plan_outcome(conn, component, remaining=remaining) for component in components) + return tuple(_raw_replay_plan_outcome(conn, plan, remaining=remaining) for plan in plans) def _raw_materialization_bucket_summary( @@ -5555,6 +5507,7 @@ class RepairResult: detail: str = "" metrics: dict[str, float] = field(default_factory=dict) plan_outcomes: tuple[RawReplayPlanOutcome, ...] = () + census_receipt: RawAuthorityCensusReceipt | None = None def to_dict(self) -> JSONDocument: payload: dict[str, object] = { @@ -5568,6 +5521,19 @@ def to_dict(self) -> JSONDocument: } if self.plan_outcomes: payload["plan_outcomes"] = [outcome.to_dict() for outcome in self.plan_outcomes] + if self.census_receipt is not None: + payload["census"] = { + "census_id": self.census_receipt.census_id, + "sequence_no": self.census_receipt.sequence_no, + "inventory_digest": self.census_receipt.inventory_digest, + "residual_digest": self.census_receipt.residual_digest, + "plan_count": self.census_receipt.plan_count, + "executable_plan_count": self.census_receipt.executable_plan_count, + "residual_plan_count": self.census_receipt.residual_plan_count, + "predecessor_census_id": self.census_receipt.predecessor_census_id, + "fixed_point": self.census_receipt.fixed_point, + "query_handle": self.census_receipt.query_handle, + } return json_document(payload) @@ -5766,6 +5732,7 @@ def _internal_derived_repair_result( detail: str, metrics: dict[str, float] | None = None, plan_outcomes: tuple[RawReplayPlanOutcome, ...] = (), + census_receipt: RawAuthorityCensusReceipt | None = None, ) -> RepairResult: return RepairResult( name=name, @@ -5776,6 +5743,7 @@ def _internal_derived_repair_result( detail=detail, metrics=dict(metrics or {}), plan_outcomes=plan_outcomes, + census_receipt=census_receipt, ) @@ -6381,6 +6349,18 @@ def repair_raw_materialization( progress_callback: ProgressCallback | None = None, ) -> RepairResult: """Converge retained raws through typed per-session revision authority.""" + archive_root = _raw_materialization_archive_root(config) + blocker_count = unresolved_raw_authority_blockers(archive_root) + if blocker_count: + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=( + f"Raw materialization is fail-closed behind {blocker_count:,} unresolved durable stale-plan blocker(s)" + ), + metrics={"raw_materialization_unresolved_blocker_count": float(blocker_count)}, + ) candidates = _raw_materialization_candidate_ids( config, raw_artifact_id=raw_artifact_id, @@ -6388,9 +6368,38 @@ def repair_raw_materialization( source_family=source_family, source_root=source_root, ) + census_failed_raw_ids: set[str] = set() + if candidates.raw_ids: + from polylogue.sources.revision_backfill import census_historical_revision_evidence + + preliminary_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + preliminary_selected = ( + preliminary_components[:raw_artifact_limit] if raw_artifact_limit is not None else preliminary_components + ) + for component in preliminary_selected: + seed = _raw_materialization_component_seed(candidates, component) + try: + census_historical_revision_evidence( + archive_root, + selected_raw_ids=[seed], + max_payload_bytes=RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES, + ) + except Exception: + # The immutable plan below conserves the component as retryable; + # a census failure must not make it disappear from inventory. + logger.exception("raw authority census failed for component containing %s", seed) + census_failed_raw_ids.update(component) + candidates = _raw_materialization_candidate_ids( + config, + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + ) candidate_raw_ids = candidates.raw_ids - archive_root = _raw_materialization_archive_root(config) ordered_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + plans = build_raw_replay_plans(archive_root, ordered_components) + plan_by_component = {plan.input_raw_ids: plan for plan in plans} all_blocked_components = [ component for component in ordered_components @@ -6402,22 +6411,33 @@ def repair_raw_materialization( ordered_components[:raw_artifact_limit] if raw_artifact_limit is not None else ordered_components ) blocked_components = [ - component for component in selected_components if all_blocked_component_raw_ids.intersection(component) + component + for component in selected_components + if all_blocked_component_raw_ids.intersection(component) or census_failed_raw_ids.intersection(component) ] blocked_component_raw_ids = {raw_id for component in blocked_components for raw_id in component} blocked_plan_outcomes = tuple( RawReplayPlanOutcome( - _raw_replay_plan_id(component), + plan_by_component[component].plan_id, component, RawReplayPlanStatus.RETRYABLE, - "authority component exceeds the bounded replay resource envelope", - "retry the same plan after streaming/resource admission is available", + ( + "authority component census did not complete" + if census_failed_raw_ids.intersection(component) + else "authority component exceeds the bounded replay resource envelope" + ), + ( + "resume the bounded source census before replay planning" + if census_failed_raw_ids.intersection(component) + else "retry the same plan after streaming/resource admission is available" + ), ) for component in blocked_components ) executable_components = [ component for component in selected_components if not blocked_component_raw_ids.intersection(component) ] + executable_plans = [plan_by_component[component] for component in executable_components] raw_ids = [_raw_materialization_component_seed(candidates, component) for component in executable_components] selected_component_raw_ids = {raw_id for component in selected_components for raw_id in component} selected_candidate_raw_ids = selected_component_raw_ids.intersection(candidate_raw_ids) @@ -6452,6 +6472,8 @@ def repair_raw_materialization( metrics["raw_materialization_before_component_count"] = float(len(ordered_components)) metrics["raw_materialization_selected_executable_component_count"] = float(len(executable_components)) metrics["raw_materialization_selected_blocked_component_count"] = float(len(blocked_components)) + if census_failed_raw_ids: + metrics["raw_materialization_census_incomplete_raw_count"] = float(len(census_failed_raw_ids)) if all_blocked_component_raw_ids: metrics["raw_materialization_resource_blocked_count"] = float(len(all_blocked_component_raw_ids)) metrics["raw_materialization_execute_blob_limit_bytes"] = float(RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES) @@ -6476,6 +6498,32 @@ def repair_raw_materialization( metrics["raw_materialization_execute_blob_limit_bytes"] = float(RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES) if oversized_stream_safe_raw_ids: metrics["raw_materialization_stream_oversized_count"] = float(len(oversized_stream_safe_raw_ids)) + census_receipt: RawAuthorityCensusReceipt | None = None + if not dry_run: + census_receipt = record_raw_authority_census( + archive_root, + plans, + selected_plan_ids={plan_by_component[component].plan_id for component in selected_components}, + scope={ + "raw_artifact_id": raw_artifact_id, + "provider": provider, + "source_family": source_family, + "source_root": str(source_root) if source_root is not None else None, + "raw_artifact_limit": raw_artifact_limit, + }, + residual={ + "missing_blobs": missing_blobs, + "missing_blob_source_available": candidates.missing_blob_source_available, + "missing_blob_source_missing": candidates.missing_blob_source_missing, + "adoption_deferred": candidates.adoption_deferred, + "authority_quarantined": candidates.authority_quarantined, + "byte_authority_fragments": candidates.byte_authority_fragments, + "byte_authority_quarantined": candidates.byte_authority_quarantined, + "byte_authority_pending": candidates.byte_authority_pending, + }, + ) + metrics["raw_materialization_census_sequence"] = float(census_receipt.sequence_no) + metrics["raw_materialization_census_fixed_point"] = float(census_receipt.fixed_point) if not candidate_raw_ids: detail = "Executable raw replay converged" if ( @@ -6515,12 +6563,13 @@ def repair_raw_materialization( ), detail=detail, metrics=metrics, + census_receipt=census_receipt, ) if dry_run: plan_outcomes = ( tuple( RawReplayPlanOutcome( - _raw_replay_plan_id(component), + plan_by_component[component].plan_id, component, RawReplayPlanStatus.RETRYABLE, "dry-run census selected this executable authority component", @@ -6554,6 +6603,51 @@ def repair_raw_materialization( detail=detail, metrics=metrics, plan_outcomes=plan_outcomes, + census_receipt=census_receipt, + ) + + if census_receipt is None: + raise RuntimeError("apply-mode raw materialization requires a durable census receipt") + for outcome in blocked_plan_outcomes: + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + + stale_outcomes: list[RawReplayPlanOutcome] = [] + validated_plans: list[RawReplayPlan] = [] + for plan in executable_plans: + valid, observed = validate_raw_replay_plan(archive_root, plan) + if valid: + validated_plans.append(plan) + else: + stale_outcomes.append(reject_stale_raw_replay_plan(archive_root, census_receipt.census_id, plan, observed)) + if stale_outcomes: + carried = [ + RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.CARRIED_FORWARD, + "another selected plan failed immutable precondition validation", + "resolve the durable stale-plan blocker before retrying this unchanged plan", + ) + for plan in validated_plans + ] + for outcome in carried: + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + plan_outcomes = tuple(stale_outcomes + carried) + blocked_plan_outcomes + metrics["raw_materialization_plan_rejected_stale_count"] = float(len(stale_outcomes)) + metrics["raw_materialization_plan_carried_forward_count"] = float(len(carried)) + metrics["raw_materialization_plan_outcome_count"] = float(len(plan_outcomes)) + metrics["raw_materialization_plan_conservation_error_count"] = float(len(stale_outcomes)) + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=( + f"Rejected {len(stale_outcomes):,} stale immutable replay plan(s); " + "automatic convergence is now fail-closed behind durable blocker evidence" + ), + metrics=metrics, + plan_outcomes=plan_outcomes, + census_receipt=census_receipt, ) from polylogue.sources.revision_backfill import ( @@ -6566,7 +6660,8 @@ def repair_raw_materialization( metrics["raw_materialization_executed_count"] = float(len(executable_raw_ids)) replay_parts: list[RevisionBackfillResult] = [] execution_outcomes: list[RawReplayPlanOutcome] = [] - for component, raw_id in zip(executable_components, executable_raw_ids, strict=True): + for plan, raw_id in zip(executable_plans, executable_raw_ids, strict=True): + component = plan.input_raw_ids try: part = backfill_historical_revision_evidence( archive_root, @@ -6577,27 +6672,27 @@ def repair_raw_materialization( metrics["raw_materialization_resource_blocked_count"] = max( metrics.get("raw_materialization_resource_blocked_count", 0.0), float(len(exc.raw_ids)) ) - execution_outcomes.append( - RawReplayPlanOutcome( - _raw_replay_plan_id(component), - component, - RawReplayPlanStatus.RETRYABLE, - "expanded authority component exceeded the bounded replay resource envelope", - "retry the same plan after streaming/resource admission is available", - ) + outcome = RawReplayPlanOutcome( + plan.plan_id, + component, + RawReplayPlanStatus.RETRYABLE, + "expanded authority component exceeded the bounded replay resource envelope", + "retry the same plan after streaming/resource admission is available", ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + execution_outcomes.append(outcome) continue except Exception as exc: - logger.exception("raw replay plan %s failed", _raw_replay_plan_id(component)) - execution_outcomes.append( - RawReplayPlanOutcome( - _raw_replay_plan_id(component), - component, - RawReplayPlanStatus.RETRYABLE, - f"component execution raised {type(exc).__name__}: {exc}", - "retry this plan after independent components have received a turn", - ) + logger.exception("raw replay plan %s failed", plan.plan_id) + outcome = RawReplayPlanOutcome( + plan.plan_id, + component, + RawReplayPlanStatus.RETRYABLE, + f"component execution raised {type(exc).__name__}: {exc}", + "retry this plan after independent components have received a turn", ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + execution_outcomes.append(outcome) continue replay_parts.append(part) current = _raw_materialization_candidate_ids( @@ -6607,7 +6702,14 @@ def repair_raw_materialization( source_family=source_family, source_root=source_root, ) - execution_outcomes.extend(_raw_replay_plan_outcomes(archive_root, [component], remaining=current)) + component_outcomes = _raw_replay_plan_outcomes(archive_root, [plan], remaining=current) + for outcome in component_outcomes: + receipted = dataclasses.replace( + outcome, + application_receipt=raw_replay_application_receipt(archive_root, plan), + ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, receipted) + execution_outcomes.append(receipted) replay = RevisionBackfillResult( scanned=sum(part.scanned for part in replay_parts), @@ -6644,9 +6746,12 @@ def repair_raw_materialization( metrics[f"raw_materialization_plan_{status.value}_count"] = float( sum(outcome.status is status for outcome in plan_outcomes) ) - metrics["raw_materialization_plan_outcome_count"] = float(len(plan_outcomes)) + carried_forward_count = len(plans) - len(selected_components) + metrics["raw_materialization_plan_carried_forward_count"] = float(carried_forward_count) + metrics["raw_materialization_plan_outcome_count"] = float(len(plans)) metrics["raw_materialization_plan_conservation_error_count"] = float( sum(outcome.status is RawReplayPlanStatus.REJECTED_STALE for outcome in plan_outcomes) + + abs(len(selected_components) - len(plan_outcomes)) ) success = ( not remaining.raw_ids @@ -6713,6 +6818,7 @@ def repair_raw_materialization( detail=detail, metrics=metrics, plan_outcomes=plan_outcomes, + census_receipt=census_receipt, ) diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index eb94034ed5..b554a4a71b 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -9,7 +9,7 @@ from polylogue.core.enums import ArtifactSupportStatus, Origin, Provider, ValidationMode, ValidationStatus from polylogue.storage.sqlite.archive_tiers.common import check, nullable_check -SOURCE_SCHEMA_VERSION = 12 +SOURCE_SCHEMA_VERSION = 13 SOURCE_DDL = f""" CREATE TABLE IF NOT EXISTS raw_sessions ( @@ -101,6 +101,79 @@ detail TEXT NOT NULL DEFAULT '' ) STRICT; +-- Durable authority reconciliation ledger. The source tier owns this +-- evidence because index.db and ops.db are rebuildable/disposable: neither +-- can be the authority for whether an accepted replay plan was conserved. +CREATE TABLE IF NOT EXISTS raw_authority_censuses ( + census_id TEXT PRIMARY KEY, + sequence_no INTEGER NOT NULL UNIQUE CHECK(sequence_no > 0), + scope_json TEXT NOT NULL CHECK(json_valid(scope_json)), + parser_fingerprint TEXT NOT NULL, + inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), + residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), + plan_count INTEGER NOT NULL CHECK(plan_count >= 0), + executable_plan_count INTEGER NOT NULL CHECK(executable_plan_count >= 0), + residual_plan_count INTEGER NOT NULL CHECK(residual_plan_count >= 0), + predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), + fixed_point INTEGER NOT NULL DEFAULT 0 CHECK(fixed_point IN (0, 1)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + completed_at_ms INTEGER NOT NULL CHECK(completed_at_ms >= created_at_ms), + CHECK(plan_count >= executable_plan_count), + CHECK(plan_count >= residual_plan_count) +) STRICT; + +CREATE TABLE IF NOT EXISTS raw_authority_plans ( + plan_id TEXT PRIMARY KEY, + input_digest TEXT NOT NULL CHECK(length(input_digest) = 64), + input_raw_ids_json TEXT NOT NULL CHECK(json_valid(input_raw_ids_json)), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + authority_witness_json TEXT NOT NULL CHECK(json_valid(authority_witness_json)), + source_preconditions_json TEXT NOT NULL CHECK(json_valid(source_preconditions_json)), + index_preconditions_json TEXT NOT NULL CHECK(json_valid(index_preconditions_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) +) STRICT; + +CREATE TABLE IF NOT EXISTS raw_authority_census_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + selected INTEGER NOT NULL CHECK(selected IN (0, 1)), + outcome_status TEXT NOT NULL CHECK(outcome_status IN ( + 'executed', 'retryable', 'deferred', 'terminal', + 'rejected_stale', 'carried_forward' + )), + reason TEXT NOT NULL, + next_action TEXT NOT NULL, + application_receipt_json TEXT NOT NULL DEFAULT '{{}}' CHECK(json_valid(application_receipt_json)), + recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_raw_authority_census_plans_status +ON raw_authority_census_plans(census_id, outcome_status, ordinal); + +CREATE INDEX IF NOT EXISTS idx_raw_authority_census_plans_attempts +ON raw_authority_census_plans(plan_id, recorded_at_ms DESC) +WHERE selected = 1; + +CREATE TABLE IF NOT EXISTS raw_authority_blockers ( + blocker_id TEXT PRIMARY KEY, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id), + reason TEXT NOT NULL, + expected_json TEXT NOT NULL CHECK(json_valid(expected_json)), + observed_json TEXT NOT NULL CHECK(json_valid(observed_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + resolved_at_ms INTEGER CHECK(resolved_at_ms IS NULL OR resolved_at_ms >= created_at_ms), + resolution TEXT, + CHECK((resolved_at_ms IS NULL) = (resolution IS NULL)) +) STRICT; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_authority_blockers_open_plan +ON raw_authority_blockers(plan_id) +WHERE resolved_at_ms IS NULL; + CREATE TABLE IF NOT EXISTS blob_refs ( blob_hash BLOB NOT NULL CHECK(length(blob_hash) = 32), ref_id TEXT NOT NULL, diff --git a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql new file mode 100644 index 0000000000..012fa0ac78 --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql @@ -0,0 +1,71 @@ +-- migration-safety: additive-no-backup +-- Durable, restart-safe conservation ledger for raw authority reconciliation. +CREATE TABLE raw_authority_censuses ( + census_id TEXT PRIMARY KEY, + sequence_no INTEGER NOT NULL UNIQUE CHECK(sequence_no > 0), + scope_json TEXT NOT NULL CHECK(json_valid(scope_json)), + parser_fingerprint TEXT NOT NULL, + inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), + residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), + plan_count INTEGER NOT NULL CHECK(plan_count >= 0), + executable_plan_count INTEGER NOT NULL CHECK(executable_plan_count >= 0), + residual_plan_count INTEGER NOT NULL CHECK(residual_plan_count >= 0), + predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), + fixed_point INTEGER NOT NULL DEFAULT 0 CHECK(fixed_point IN (0, 1)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + completed_at_ms INTEGER NOT NULL CHECK(completed_at_ms >= created_at_ms), + CHECK(plan_count >= executable_plan_count), + CHECK(plan_count >= residual_plan_count) +) STRICT; + +CREATE TABLE raw_authority_plans ( + plan_id TEXT PRIMARY KEY, + input_digest TEXT NOT NULL CHECK(length(input_digest) = 64), + input_raw_ids_json TEXT NOT NULL CHECK(json_valid(input_raw_ids_json)), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + authority_witness_json TEXT NOT NULL CHECK(json_valid(authority_witness_json)), + source_preconditions_json TEXT NOT NULL CHECK(json_valid(source_preconditions_json)), + index_preconditions_json TEXT NOT NULL CHECK(json_valid(index_preconditions_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) +) STRICT; + +CREATE TABLE raw_authority_census_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + selected INTEGER NOT NULL CHECK(selected IN (0, 1)), + outcome_status TEXT NOT NULL CHECK(outcome_status IN ( + 'executed', 'retryable', 'deferred', 'terminal', + 'rejected_stale', 'carried_forward' + )), + reason TEXT NOT NULL, + next_action TEXT NOT NULL, + application_receipt_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(application_receipt_json)), + recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + +CREATE INDEX idx_raw_authority_census_plans_status +ON raw_authority_census_plans(census_id, outcome_status, ordinal); + +CREATE INDEX idx_raw_authority_census_plans_attempts +ON raw_authority_census_plans(plan_id, recorded_at_ms DESC) +WHERE selected = 1; + +CREATE TABLE raw_authority_blockers ( + blocker_id TEXT PRIMARY KEY, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id), + reason TEXT NOT NULL, + expected_json TEXT NOT NULL CHECK(json_valid(expected_json)), + observed_json TEXT NOT NULL CHECK(json_valid(observed_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + resolved_at_ms INTEGER CHECK(resolved_at_ms IS NULL OR resolved_at_ms >= created_at_ms), + resolution TEXT, + CHECK((resolved_at_ms IS NULL) = (resolution IS NULL)) +) STRICT; + +CREATE UNIQUE INDEX idx_raw_authority_blockers_open_plan +ON raw_authority_blockers(plan_id) +WHERE resolved_at_ms IS NULL; diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index c6a902de57..c21f7ca8f5 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -644,6 +644,53 @@ def test_raw_materialization_pass_emits_zero_work_receipt(monkeypatch: pytest.Mo assert events[0][1]["plan_outcomes"] == [] +def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: pytest.MonkeyPatch) -> None: + from polylogue.daemon import cli as daemon_cli + + events: list[tuple[str, dict[str, object]]] = [] + monkeypatch.setattr( + "polylogue.daemon.events.emit_daemon_event", + lambda kind, *, payload: events.append((kind, payload)), + ) + monkeypatch.setattr(os, "urandom", lambda _size: b"c" * 16) + census = SimpleNamespace( + census_id="census:2:inventory:residual", + sequence_no=2, + inventory_digest="a" * 64, + residual_digest="b" * 64, + plan_count=3, + executable_plan_count=1, + residual_plan_count=2, + predecessor_census_id="census:1:inventory:residual", + fixed_point=False, + query_handle="raw-authority-census:census:2:inventory:residual", + ) + + daemon_cli._emit_raw_materialization_pass( + SimpleNamespace( + success=False, + repaired_count=0, + detail="bounded", + metrics={}, + plan_outcomes=(), + census_receipt=census, + ) + ) + + assert events[0][1]["census"] == { + "census_id": census.census_id, + "sequence_no": 2, + "inventory_digest": "a" * 64, + "residual_digest": "b" * 64, + "plan_count": 3, + "executable_plan_count": 1, + "residual_plan_count": 2, + "predecessor_census_id": census.predecessor_census_id, + "fixed_point": False, + "query_handle": census.query_handle, + } + + def test_raw_materialization_closes_fts_on_cancellation( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index 4527a35734..e125d16fc9 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -489,7 +489,7 @@ def test_source_tier_v1_migrates_to_current_without_native_uniqueness( result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) assert result.from_version == 1 assert result.to_version == SOURCE_SCHEMA_VERSION - assert result.applied_versions == (2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) + assert result.applied_versions == (2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13) assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == SOURCE_SCHEMA_VERSION columns = {str(row[1]) for row in conn.execute("PRAGMA table_info('raw_sessions')")} assert "predecessor_source_revision" in columns @@ -553,6 +553,10 @@ def test_source_publication_backfill_requires_verified_backup( ) -> None: db_path = workspace_env["archive_root"] / "source.db" with sqlite3.connect(db_path) as conn: + conn.execute("DROP TABLE raw_authority_blockers") + conn.execute("DROP TABLE raw_authority_census_plans") + conn.execute("DROP TABLE raw_authority_plans") + conn.execute("DROP TABLE raw_authority_censuses") conn.execute("DROP TABLE excised_content") conn.execute("DROP TABLE sinex_publication_segments") conn.execute("DROP TABLE sinex_publication_receipts") @@ -569,8 +573,8 @@ def test_source_publication_backfill_requires_verified_backup( result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) assert result.from_version == 9 - assert result.to_version == SOURCE_SCHEMA_VERSION == 12 - assert result.applied_versions == (10, 11, 12) + assert result.to_version == SOURCE_SCHEMA_VERSION == 13 + assert result.applied_versions == (10, 11, 12, 13) assert result.backup_receipt == manifest.with_name("verification-receipt.json") tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} assert { @@ -579,6 +583,10 @@ def test_source_publication_backfill_requires_verified_backup( "sinex_publication_segments", "sinex_publication_receipts", "excised_content", + "raw_authority_censuses", + "raw_authority_plans", + "raw_authority_census_plans", + "raw_authority_blockers", } <= tables @@ -643,6 +651,9 @@ def test_source_tier_v7_expands_origin_checks_with_verified_backup( " CHECK(revision_authority IN ('asserted', 'byte_proven', 'quarantined'))\n" " ,predecessor_source_revision TEXT\n", ) + authority_start = old_ddl.index("-- Durable authority reconciliation ledger.") + authority_end = old_ddl.index("CREATE TABLE IF NOT EXISTS blob_refs", authority_start) + old_ddl = old_ddl[:authority_start] + old_ddl[authority_end:] # Migration 010 adds `excised_content` (polylogue-27m) -- a v7 snapshot # predates it, same as it predates the beads-origin/capture_mode diffs # stripped above. Without this, the fixture (built from the CURRENT @@ -715,8 +726,8 @@ def test_source_tier_v7_expands_origin_checks_with_verified_backup( with sqlite3.connect(db_path) as conn: result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) assert result.from_version == 7 - assert result.to_version == SOURCE_SCHEMA_VERSION == 12 - assert result.applied_versions == (8, 9, 10, 11, 12) + assert result.to_version == SOURCE_SCHEMA_VERSION == 13 + assert result.applied_versions == (8, 9, 10, 11, 12, 13) assert conn.execute( """ SELECT predecessor_source_revision, predecessor_raw_id, baseline_raw_id, @@ -877,7 +888,7 @@ def test_source_tier_v2_migrates_to_v3_dropping_pending_blob_refs( assert result.from_version == 2 assert result.to_version == SOURCE_SCHEMA_VERSION - assert result.applied_versions == (3, 4, 5, 6, 7, 8, 9, 10, 11, 12) + assert result.applied_versions == (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13) assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == SOURCE_SCHEMA_VERSION assert not conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name='pending_blob_refs'" @@ -935,7 +946,7 @@ def test_source_tier_v3_adds_publication_reservations_with_verified_backup_recei conn = sqlite3.connect(db_path) try: result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) - assert result.applied_versions == (4, 5, 6, 7, 8, 9, 10, 11, 12) + assert result.applied_versions == (4, 5, 6, 7, 8, 9, 10, 11, 12, 13) assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == SOURCE_SCHEMA_VERSION conn.execute( """ diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py new file mode 100644 index 0000000000..63b38c91cb --- /dev/null +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from typing import cast + +from polylogue.config import Config +from polylogue.core.enums import Provider +from polylogue.sources.revision_backfill import census_historical_revision_evidence +from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot +from polylogue.storage.raw_authority import ( + build_raw_replay_plans, + record_raw_authority_census, + reject_stale_raw_replay_plan, + validate_raw_replay_plan, +) +from polylogue.storage.repair import repair_raw_materialization +from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + +def _config(root: Path) -> Config: + return Config(archive_root=root, render_root=root / "render", sources=[], db_path=root / "archive.db") + + +def _write_codex_raw( + root: Path, + *, + native_id: str, + source_path: str, + acquired_at_ms: int, + text: str = "", +) -> str: + payload = ( + f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\n' + f'{{"type":"response_item","payload":{{"type":"message","id":"m-{acquired_at_ms}",' + f'"role":"user","content":[{{"type":"input_text","text":"{text}"}}]}}}}\n' + ).encode() + with ArchiveStore.open_existing(root, read_only=False) as archive: + return archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=source_path, + acquired_at_ms=acquired_at_ms, + ) + + +def test_moved_path_census_stabilizes_preview_and_apply_plan_identity(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + old_raw = _write_codex_raw( + tmp_path, + native_id="moved-session", + source_path="old/location.jsonl", + acquired_at_ms=2, + text="old", + ) + new_raw = _write_codex_raw( + tmp_path, + native_id="moved-session", + source_path="new/location.jsonl", + acquired_at_ms=1, + text="new", + ) + # Prior history already knows the logical key at another path. The new + # raw begins as an uncensused singleton and must discover that history + # before an immutable plan is assigned. + census_historical_revision_evidence(tmp_path, selected_raw_ids=[old_raw]) + + preview = repair_raw_materialization(_config(tmp_path), dry_run=True, raw_artifact_limit=1) + applied = repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) + + assert len(preview.plan_outcomes) == len(applied.plan_outcomes) == 1 + assert preview.plan_outcomes[0].plan_id == applied.plan_outcomes[0].plan_id + assert set(preview.plan_outcomes[0].input_raw_ids) == {old_raw, new_raw} + assert set(applied.plan_outcomes[0].input_raw_ids) == {old_raw, new_raw} + + +def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="first", source_path="first.jsonl", acquired_at_ms=1) + _write_codex_raw(tmp_path, native_id="second", source_path="second.jsonl", acquired_at_ms=2) + + result = repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) + + assert result.census_receipt is not None + assert result.census_receipt.plan_count == 2 + assert result.census_receipt.executable_plan_count == 1 + assert result.census_receipt.residual_plan_count == 1 + assert result.metrics["raw_materialization_plan_outcome_count"] == 2.0 + assert result.metrics["raw_materialization_plan_carried_forward_count"] == 1.0 + with sqlite3.connect(tmp_path / "source.db") as conn: + rows = conn.execute( + """ + SELECT selected, outcome_status, application_receipt_json + FROM raw_authority_census_plans + WHERE census_id = ? ORDER BY ordinal + """, + (result.census_receipt.census_id,), + ).fetchall() + assert {row[1] for row in rows} == {"executed", "carried_forward"} + executed = next(row for row in rows if row[1] == "executed") + assert executed[0] == 1 + assert '"application_rows"' in executed[2] + readiness = raw_materialization_readiness_snapshot(tmp_path) + census_status = cast(dict[str, object], readiness["raw_authority_census"]) + assert census_status["census_id"] == result.census_receipt.census_id + assert census_status["inventory_digest"] == result.census_receipt.inventory_digest + assert census_status["residual_digest"] == result.census_receipt.residual_digest + assert census_status["plan_count"] == 2 + assert census_status["executable_plan_count"] == 1 + assert census_status["residual_plan_count"] == 1 + assert census_status["query_handle"] == result.census_receipt.query_handle + + +def test_two_successive_quiescent_censuses_are_required_for_fixed_point(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="fixed", source_path="fixed.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).repaired_count == 1 + + first_empty = repair_raw_materialization(_config(tmp_path)) + second_empty = repair_raw_materialization(_config(tmp_path)) + + assert first_empty.census_receipt is not None + assert second_empty.census_receipt is not None + assert first_empty.census_receipt.fixed_point is False + assert second_empty.census_receipt.fixed_point is True + assert first_empty.census_receipt.inventory_digest == second_empty.census_receipt.inventory_digest + assert first_empty.census_receipt.residual_digest == second_empty.census_receipt.residual_digest + + +def test_stale_plan_persists_blocker_before_automatic_replay_refuses_work(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="stale", source_path="stale.jsonl", acquired_at_ms=1) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + scope={"test": "stale"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET source_path = 'moved-after-plan.jsonl' WHERE raw_id = ?", (raw_id,)) + conn.commit() + + valid, observed = validate_raw_replay_plan(tmp_path, plan) + assert valid is False + outcome = reject_stale_raw_replay_plan(tmp_path, census.census_id, plan, observed) + + assert outcome.status.value == "rejected_stale" + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + assert ( + conn.execute( + "SELECT outcome_status FROM raw_authority_census_plans WHERE census_id = ? AND plan_id = ?", + (census.census_id, plan.plan_id), + ).fetchone()[0] + == "rejected_stale" + ) + refused = repair_raw_materialization(_config(tmp_path)) + assert refused.success is False + assert refused.metrics["raw_materialization_unresolved_blocker_count"] == 1.0 diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 4ea5adbd21..6e60a01e1b 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import sqlite3 from collections.abc import Iterator from contextlib import contextmanager @@ -1153,12 +1152,6 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) - with sqlite3.connect(tmp_path / "source.db") as source_conn: - source_authority_before = source_conn.execute( - "SELECT revision_authority FROM raw_sessions WHERE raw_id = ?", - (raw_id,), - ).fetchone() - result = repair_mod.repair_raw_materialization(config, dry_run=False) assert result.success is False @@ -1188,7 +1181,10 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: "SELECT parsed_at_ms, parse_error, revision_authority FROM raw_sessions WHERE raw_id = ?", (raw_id,), ).fetchone() - assert raw_state == (None, None, source_authority_before[0]) + # The source-only census now completes before replay planning. It may + # establish byte-proven source authority, while the incomparable index + # state still remains untouched and receives a deferred application. + assert raw_state == (None, None, "byte_proven") with sqlite3.connect(tmp_path / "index.db") as index_conn: deferred = index_conn.execute( "SELECT decision, detail FROM raw_revision_applications WHERE raw_id = ?", @@ -1213,24 +1209,27 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: def test_raw_materialization_dry_run_reports_limited_selection( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + sizes = [512, 1024, 2048, 4096] + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_ids = [ + archive.write_raw_payload( + provider=Provider.CODEX, + payload=f'{{"type":"session_meta","payload":{{"id":"dry-{index}"}}}}\n'.encode(), + source_path=f"dry-{index}.jsonl", + acquired_at_ms=index + 1, + ) + for index in range(4) + ] + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.executemany("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", zip(sizes, raw_ids, strict=True)) + conn.commit() config = _config(tmp_path) - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-slow", "raw-2", "raw-3", "raw-4"], - 0, - 4, - { - "raw-slow": 512, - "raw-2": 1024, - "raw-3": 2048, - "raw-4": 4096, - }, - ), - ) result = repair_mod.repair_raw_materialization( config, @@ -1251,52 +1250,29 @@ def test_raw_materialization_dry_run_reports_limited_selection( def test_raw_materialization_execute_limits_authority_selection( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - config = _config(tmp_path) - calls: dict[str, object] = {} - - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - calls["db_path"] = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - calls["archive_root"] = archive_root - - async def close(self) -> None: - calls["closed"] = True - - class FakeParseResult: - processed_ids = {"session-2", "session-3"} - parse_failures = 0 - - class FakeParsingService: - def __init__(self, **_kwargs: object) -> None: - pass - - async def parse_from_raw(self, **kwargs: object) -> FakeParseResult: - calls["parse_kwargs"] = kwargs - return FakeParseResult() + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-slow", "raw-2", "raw-3", "raw-4"], - 0, - 4, - { - "raw-slow": 512, - "raw-2": 1024, - "raw-3": 2048, - "raw-4": 4096, - }, - ), - ) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_ids = [ + archive.write_raw_payload( + provider=Provider.CODEX, + payload=f'{{"type":"session_meta","payload":{{"id":"execute-{index}"}}}}\n'.encode(), + source_path=f"execute-{index}.jsonl", + acquired_at_ms=index + 1, + ) + for index in range(4) + ] + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.executemany( + "UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", + zip((512, 1024, 2048, 4096), raw_ids, strict=True), + ) + conn.commit() + config = _config(tmp_path) result = repair_mod.repair_raw_materialization( config, @@ -1304,8 +1280,7 @@ async def parse_from_raw(self, **kwargs: object) -> FakeParseResult: ) assert result.success is False - assert result.repaired_count == 0 - assert "parse_kwargs" not in calls + assert result.repaired_count == 2 assert result.metrics["raw_materialization_candidate_count"] == 4.0 assert result.metrics["raw_materialization_selected_count"] == 2.0 assert result.metrics["raw_materialization_executed_count"] == 2.0 @@ -1564,116 +1539,66 @@ def test_raw_materialization_uses_authority_substrate_not_legacy_ingest_stage( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - config = _config(tmp_path) - calls: dict[str, object] = {} - - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - calls["db_path"] = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - calls["archive_root"] = archive_root - - async def close(self) -> None: - calls["closed"] = True + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root - class FakeParseResult: - processed_ids = {"session-1", "session-2"} - parse_failures = 0 + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"authority-substrate"}}\n', + source_path="authority-substrate.jsonl", + acquired_at_ms=1, + ) + config = _config(tmp_path) - class FakeParsingService: + class UnexpectedParsingService: def __init__(self, **_kwargs: object) -> None: - pass - - async def parse_from_raw(self, **kwargs: object) -> FakeParseResult: - calls["parse_kwargs"] = kwargs - return FakeParseResult() + pytest.fail("raw authority repair must not construct the legacy ParsingService") - def fake_candidate_ids( - _config: Config, - *, - raw_artifact_id: str | None = None, - provider: str | None = None, - source_family: str | None = None, - source_root: Path | None = None, - ) -> repair_mod.RawMaterializationCandidates: - calls["raw_artifact_id"] = raw_artifact_id - calls["provider"] = provider - calls["source_family"] = source_family - calls["source_root"] = source_root - return repair_mod.RawMaterializationCandidates(["raw-1"], 0, 0) - - monkeypatch.setattr(repair_mod, "_raw_materialization_candidate_ids", fake_candidate_ids) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) + monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) result = repair_mod.repair_raw_materialization(config, dry_run=False) - assert result.success is False - assert result.repaired_count == 0 + assert result.success is True + assert result.repaired_count == 1 assert "typed revision authority" in result.detail - assert "parse_kwargs" not in calls - assert calls["raw_artifact_id"] is None - assert "closed" not in calls def test_raw_materialization_reports_authority_progress_and_payload_size( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"progress"}}\n', + source_path="progress.jsonl", + acquired_at_ms=1, + ) + declared_size = 256 * 1024 * 1024 + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", (declared_size, raw_id)) + conn.commit() config = _config(tmp_path) progress: list[str] = [] - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - self.db_path = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - self.backend = backend - self.archive_root = archive_root - - async def close(self) -> None: - pass - - class FakeParseResult: - processed_ids = {"session-1"} - parse_failures = 0 - - class FakeParsingService: - def __init__(self, **_kwargs: object) -> None: - pass - - async def parse_from_raw(self, **_kwargs: object) -> FakeParseResult: - return FakeParseResult() - - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-1"], - 0, - 0, - {"raw-1": 256 * 1024 * 1024}, - ), - ) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) - result = repair_mod.repair_raw_materialization( config, dry_run=False, progress_callback=lambda _amount, desc=None: progress.append(desc or ""), ) - assert result.success is False + assert result.success is True assert len(progress) == 1 assert "typed revision authority" in progress[0] - assert result.metrics["raw_materialization_total_blob_bytes"] == float(256 * 1024 * 1024) - assert result.metrics["raw_materialization_max_blob_bytes"] == float(256 * 1024 * 1024) + assert result.metrics["raw_materialization_total_blob_bytes"] == float(declared_size) + assert result.metrics["raw_materialization_max_blob_bytes"] == float(declared_size) assert result.metrics["raw_materialization_selected_count"] == 1.0 @@ -1681,25 +1606,28 @@ def test_raw_materialization_blocks_oversized_actual_replay( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CHATGPT, + payload=b"{}", + source_path="oversized.json", + acquired_at_ms=1, + ) + oversized = 2 * 1024 * 1024 * 1024 + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", (oversized, raw_id)) + conn.commit() config = _config(tmp_path) class UnexpectedParsingService: def __init__(self, **_kwargs: object) -> None: raise AssertionError("oversized raw rows should be blocked before parsing") - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-1"], - 0, - 0, - {"raw-1": 2 * 1024 * 1024 * 1024}, - expanded_raw_ids=("raw-1",), - expanded_blob_bytes={"raw-1": 2 * 1024 * 1024 * 1024}, - authority_components=(("raw-1",),), - ), - ) monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) result = repair_mod.repair_raw_materialization(config, dry_run=False) @@ -1717,6 +1645,22 @@ def test_raw_materialization_classifies_oversized_stream_record_replay( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"oversized-stream"}}\n', + source_path="/captures/codex/session.jsonl", + acquired_at_ms=1, + ) + oversized = 2 * 1024 * 1024 * 1024 + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", (oversized, raw_id)) + conn.commit() config = _config(tmp_path) calls: dict[str, object] = {} @@ -1743,21 +1687,6 @@ async def parse_from_raw(self, **kwargs: object) -> FakeParseResult: calls["parse_kwargs"] = kwargs return FakeParseResult() - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-1"], - 0, - 0, - {"raw-1": 2 * 1024 * 1024 * 1024}, - {"raw-1": "claude-code-session"}, - {"raw-1": "/captures/claude/session.jsonl"}, - expanded_raw_ids=("raw-1",), - expanded_blob_bytes={"raw-1": 2 * 1024 * 1024 * 1024}, - authority_components=(("raw-1",),), - ), - ) monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) @@ -1959,8 +1888,8 @@ def test_raw_materialization_processes_independent_components_across_bounded_pas assert repair_mod.repair_raw_materialization(config, raw_artifact_limit=5).success is True -def test_raw_materialization_receipts_rotate_retryable_plan_behind_unattempted_work(tmp_path: Path) -> None: - """A retryable oldest component must not monopolize a bounded daemon slot.""" +def test_raw_materialization_durable_ledger_survives_ops_reset_for_fairness(tmp_path: Path) -> None: + """A retryable oldest component must not monopolize a slot after ops reset.""" from polylogue.core.enums import Provider from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -1988,12 +1917,7 @@ def test_raw_materialization_receipts_rotate_retryable_plan_behind_unattempted_w first = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) assert first.plan_outcomes[0].status.value == "retryable" - with sqlite3.connect(tmp_path / "ops.db") as conn: - conn.execute( - "INSERT INTO daemon_events(ts_ms, kind, payload_json) VALUES (1, 'raw_materialization_pass', ?)", - (json.dumps({"plan_outcomes": [outcome.to_dict() for outcome in first.plan_outcomes]}),), - ) - conn.commit() + (tmp_path / "ops.db").unlink() second = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) assert second.repaired_count == 1 @@ -2101,7 +2025,8 @@ def test_raw_materialization_batch_limit_counts_authority_components(tmp_path: P assert first.repaired_count == 3, (first.detail, first.metrics, after) assert first.metrics["raw_materialization_selected_component_count"] == 3.0 - assert first.metrics["raw_materialization_plan_outcome_count"] == 3.0 + assert first.metrics["raw_materialization_plan_outcome_count"] == 5.0 + assert first.metrics["raw_materialization_plan_carried_forward_count"] == 2.0 assert first.metrics["raw_materialization_plan_executed_count"] == 3.0 assert {outcome.plan_id for outcome in first.plan_outcomes} == { outcome.plan_id for outcome in preview.plan_outcomes @@ -2114,39 +2039,32 @@ def test_raw_materialization_quarantines_parse_failures_without_legacy_parser( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - config = _config(tmp_path) - - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - self.db_path = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - self.backend = backend - self.archive_root = archive_root - - async def close(self) -> None: - pass + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root - class FakeParseResult: - processed_ids: set[str] = set() - parse_failures = 1 + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b"\xff\n", + source_path="broken.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET parsed_at_ms = 1 WHERE raw_id = ?", (raw_id,)) + conn.commit() + config = _config(tmp_path) - class FakeParsingService: + class UnexpectedParsingService: def __init__(self, **_kwargs: object) -> None: - pass - - async def parse_from_raw(self, **_kwargs: object) -> FakeParseResult: - return FakeParseResult() + pytest.fail("parse failures must remain inside the authority census route") + monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates(["raw-1"], 0, 1), + "polylogue.sources.revision_backfill._parse_retained_raw", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("synthetic retained-byte decode failure")), ) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) result = repair_mod.repair_raw_materialization(config, dry_run=False) From cebd03c5d393fece37f013c6b29132f45177b150 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 16 Jul 2026 22:25:12 +0200 Subject: [PATCH 02/13] docs: refresh topology after master rebase Regenerate the topology projection after rebasing the raw-authority module onto current master. Co-Authored-By: Claude --- docs/plans/topology-target.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index 8f2c8f103c..d7af104f60 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -586,7 +586,7 @@ files: owner: archive-session reason: archive-domain semantics - path: polylogue/archive/session/extraction.py - loc: 571 + loc: 445 target: polylogue/archive/session/extraction.py owner: archive-session reason: archive-domain semantics @@ -611,7 +611,7 @@ files: owner: archive-session reason: archive-domain semantics - path: polylogue/archive/session/runtime.py - loc: 584 + loc: 587 target: polylogue/archive/session/runtime.py owner: archive-session reason: archive-domain semantics @@ -940,7 +940,7 @@ files: target: polylogue/cli/commands/scan_secrets.py owner: stable - path: polylogue/cli/commands/status.py - loc: 2522 + loc: 2551 target: polylogue/cli/commands/status.py owner: stable - path: polylogue/cli/commands/status_diagnostics.py @@ -1871,7 +1871,7 @@ files: target: polylogue/insights/resume.py owner: stable - path: polylogue/insights/rigor.py - loc: 1091 + loc: 1067 target: polylogue/insights/rigor.py owner: stable - path: polylogue/insights/run_projection.py @@ -3162,7 +3162,7 @@ files: owner: stable cross_cut: { lifecycle: model } - path: polylogue/sources/revision_backfill.py - loc: 407 + loc: 410 target: polylogue/sources/revision_backfill.py owner: stable - path: polylogue/sources/source_acquisition.py @@ -3290,7 +3290,7 @@ files: target: polylogue/storage/embeddings/embedding_stats.py owner: stable - path: polylogue/storage/embeddings/materialization.py - loc: 1327 + loc: 1330 target: polylogue/storage/embeddings/materialization.py owner: stable - path: polylogue/storage/embeddings/models.py @@ -3484,7 +3484,7 @@ files: target: polylogue/storage/raw/models.py owner: stable - path: polylogue/storage/raw_authority.py - loc: 542 + loc: 547 target: TBD owner: storage-domain - path: polylogue/storage/raw_retention.py @@ -3493,7 +3493,7 @@ files: owner: storage-root reason: storage-root cross-cutting helper - path: polylogue/storage/repair.py - loc: 7006 + loc: 7015 target: polylogue/storage/repair.py owner: storage-root reason: storage-root cross-cutting helper From 9530852cd03b5343b3cc573020114e8e31817849 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 16 Jul 2026 22:29:14 +0200 Subject: [PATCH 03/13] test(storage): prove atomic raw authority census retry Inject an abort during the second census-plan association and prove the source transaction exposes no partial census or plan rows before a clean single retry. Strengthen plan/receipt assertions to cover every witness and membership/application evidence field. Ref polylogue-hjpx.1. Co-Authored-By: Claude --- .../unit/storage/test_raw_authority_ledger.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 63b38c91cb..a673d1ea32 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import cast +import pytest + from polylogue.config import Config from polylogue.core.enums import Provider from polylogue.sources.revision_backfill import census_historical_revision_evidence @@ -97,10 +99,25 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat """, (result.census_receipt.census_id,), ).fetchall() + plan_row = conn.execute( + """ + SELECT input_raw_ids_json, logical_keys_json, authority_witness_json, + source_preconditions_json, index_preconditions_json + FROM raw_authority_plans + WHERE plan_id = ( + SELECT plan_id FROM raw_authority_census_plans + WHERE census_id = ? AND selected = 1 + ) + """, + (result.census_receipt.census_id,), + ).fetchone() assert {row[1] for row in rows} == {"executed", "carried_forward"} executed = next(row for row in rows if row[1] == "executed") assert executed[0] == 1 assert '"application_rows"' in executed[2] + assert '"membership_rows"' in executed[2] + assert plan_row is not None + assert all(value not in (None, "", "[]", "{}") for value in plan_row) readiness = raw_materialization_readiness_snapshot(tmp_path) census_status = cast(dict[str, object], readiness["raw_authority_census"]) assert census_status["census_id"] == result.census_receipt.census_id @@ -163,3 +180,60 @@ def test_stale_plan_persists_blocker_before_automatic_replay_refuses_work(tmp_pa refused = repair_raw_materialization(_config(tmp_path)) assert refused.success is False assert refused.metrics["raw_materialization_unresolved_blocker_count"] == 1.0 + + +def test_interrupted_census_has_no_partial_plan_visibility_and_retries_once(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + first = _write_codex_raw(tmp_path, native_id="atomic-first", source_path="atomic-first.jsonl", acquired_at_ms=1) + second = _write_codex_raw( + tmp_path, + native_id="atomic-second", + source_path="atomic-second.jsonl", + acquired_at_ms=2, + ) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[first, second]) + plans = build_raw_replay_plans(tmp_path, ((first,), (second,))) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + """ + CREATE TRIGGER abort_second_census_plan + BEFORE INSERT ON raw_authority_census_plans + WHEN NEW.ordinal = 1 + BEGIN + SELECT RAISE(ABORT, 'synthetic census interruption'); + END + """ + ) + conn.commit() + + with pytest.raises(sqlite3.IntegrityError, match="synthetic census interruption"): + record_raw_authority_census( + tmp_path, + plans, + selected_plan_ids={plan.plan_id for plan in plans}, + scope={"test": "interruption"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM raw_authority_plans").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM raw_authority_census_plans").fetchone()[0] == 0 + conn.execute("DROP TRIGGER abort_second_census_plan") + conn.commit() + + receipt = record_raw_authority_census( + tmp_path, + plans, + selected_plan_ids={plan.plan_id for plan in plans}, + scope={"test": "interruption"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0] == 1 + assert ( + conn.execute( + "SELECT COUNT(*) FROM raw_authority_census_plans WHERE census_id = ?", + (receipt.census_id,), + ).fetchone()[0] + == 2 + ) From f4dfd9556590cd5f9d6aeb7b70bef20b018f7ac8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 16 Jul 2026 22:47:16 +0200 Subject: [PATCH 04/13] feat(storage): expose durable raw authority censuses Problem: raw replay dry-runs did not persist the two-census fixed-point proof, and receipt query handles did not resolve to the conserved plan ledger. Daemon events also projected every outcome instead of a bounded summary. Record dry-run census receipts without advancing apply fairness, distinguish the full executable inventory from the selected batch, and expose a paginated URI through CLI and MCP. Bound daemon outcome samples while retaining counts, digests, and the ledger handle. Verification: devtools test -k raw_materialization (81 passed); revision backfill (10 passed); raw authority (9 passed); public CLI/MCP focused tests passed; devtools verify --quick passed 16/16. Ref polylogue-hjpx.1. Co-Authored-By: Claude --- docs/maintenance.md | 19 ++ .../cli/commands/maintenance/__init__.py | 6 + .../cli/commands/maintenance/_raw_identity.py | 48 +++++ polylogue/daemon/cli.py | 5 +- polylogue/mcp/server_resources.py | 24 +++ polylogue/storage/archive_readiness.py | 2 +- polylogue/storage/raw_authority.py | 178 +++++++++++++++++- polylogue/storage/repair.py | 54 +++--- tests/infra/mcp.py | 1 + .../unit/cli/test_archive_maintenance_cli.py | 35 ++++ tests/unit/daemon/test_daemon_cli.py | 35 +++- tests/unit/mcp/test_server_surfaces.py | 31 +++ .../unit/storage/test_raw_authority_ledger.py | 19 +- tests/unit/storage/test_repair.py | 2 + 14 files changed, 418 insertions(+), 41 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index ca5787cf4a..556ae27eba 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -334,6 +334,25 @@ structural (missing columns, corrupted index file, or a broken write path). Stop the daemon, restore from backup or rebuild the affected index tier, and open an issue with the probe output attached. +### Inspecting a raw-authority census + +Raw source-to-index convergence records an immutable census in `source.db`. +Status and daemon receipts expose a bounded summary plus a URI such as +`polylogue://raw-authority-census/census:42:.../0`. Resolve that same URI from +the CLI without copying every plan into the status payload: + +```bash +polylogue ops maintenance raw-authority-census \ + 'polylogue://raw-authority-census/census:42:.../0' \ + --output-format json +``` + +The response includes the census digests, complete witnesses and outcome for +the current page, any linked stale-plan blockers, and `next_query_handle` when +more rows remain. `--limit` is bounded to 1–500; `--offset` can override the +offset encoded in the URI. MCP clients resolve the URI directly through the +matching resource template. + ### Draining the convergence-debt queue **Symptoms.** `polylogue ops diagnostics workload` reports a non-trivial diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 921683bdcf..304436a1b1 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -53,6 +53,12 @@ "rebuild_index_command", "Inspect or execute an authority-safe source-to-index rebuild.", ), + ( + "raw-authority-census", + "_raw_identity", + "raw_authority_census_command", + "Read a bounded page from a durable raw-authority census ledger.", + ), ( "missing-raw-blob-cursors", "_raw_identity", diff --git a/polylogue/cli/commands/maintenance/_raw_identity.py b/polylogue/cli/commands/maintenance/_raw_identity.py index 60481ad48e..6779ecf2ec 100644 --- a/polylogue/cli/commands/maintenance/_raw_identity.py +++ b/polylogue/cli/commands/maintenance/_raw_identity.py @@ -17,6 +17,54 @@ from polylogue.paths import archive_root, render_root +@click.command("raw-authority-census") +@click.argument("query_handle") +@click.option("--limit", type=click.IntRange(1, 500), default=100, show_default=True) +@click.option("--offset", type=click.IntRange(min=0), default=None) +@click.option( + "--output-format", + "output_format", + type=click.Choice(["plain", "json"]), + default="plain", + show_default=True, +) +@click.pass_obj +def raw_authority_census_command( + env: AppEnv, + query_handle: str, + limit: int, + offset: int | None, + output_format: str, +) -> None: + """Read a bounded page from a durable raw-authority census ledger.""" + del env + from polylogue.storage.raw_authority import read_raw_authority_census + + try: + payload = read_raw_authority_census(archive_root(), query_handle, limit=limit, offset=offset) + except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + return + census = payload["census"] + if not isinstance(census, dict): + raise click.ClickException("invalid raw authority census payload") + click.echo( + f"Census {census['census_id']}: plans={census['plan_count']} " + f"executable={census['executable_plan_count']} residual={census['residual_plan_count']} " + f"fixed_point={str(census['fixed_point']).lower()}" + ) + for item in payload["plans"] if isinstance(payload["plans"], list) else []: + if isinstance(item, dict): + plan = item.get("plan") + plan_id = plan.get("plan_id") if isinstance(plan, dict) else "unknown" + click.echo(f" {item.get('ordinal')} {item.get('outcome_status')} {plan_id}") + next_handle = payload.get("next_query_handle") + if next_handle is not None: + click.echo(f"Next: {next_handle}") + + def _raw_blob_path_for_hash(root: Path, blob_hash: bytes | str) -> Path | None: hex_hash = blob_hash.hex() if isinstance(blob_hash, bytes) else str(blob_hash).lower() if len(hex_hash) != 64 or any(char not in "0123456789abcdef" for char in hex_hash): diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 899d89107f..d96c755482 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -639,6 +639,7 @@ def _drain_raw_materialization_once(*, limit: int = _RAW_MATERIALIZATION_CONVERG def _emit_raw_materialization_pass(result: Any) -> None: """Persist the conserved plan outcomes for one bounded daemon pass.""" outcomes = tuple(getattr(result, "plan_outcomes", ())) + outcome_sample_limit = 8 from polylogue.daemon.events import emit_daemon_event metrics = dict(getattr(result, "metrics", {})) @@ -663,7 +664,9 @@ def _emit_raw_materialization_pass(result: Any) -> None: "repaired_count": int(result.repaired_count), "detail": str(result.detail), "metrics": metrics, - "plan_outcomes": [outcome.to_dict() for outcome in outcomes], + "plan_outcome_count": len(outcomes), + "plan_outcome_sample": [outcome.to_dict() for outcome in outcomes[:outcome_sample_limit]], + "plan_outcome_sample_truncated": len(outcomes) > outcome_sample_limit, } if census_payload is not None: payload["census"] = census_payload diff --git a/polylogue/mcp/server_resources.py b/polylogue/mcp/server_resources.py index 1805974a2a..54a1ea5ef2 100644 --- a/polylogue/mcp/server_resources.py +++ b/polylogue/mcp/server_resources.py @@ -190,5 +190,29 @@ def readiness_resource() -> str: exclude_none=True, ) + @mcp.resource("polylogue://raw-authority-census/{census_id}/{offset}") + def raw_authority_census_resource(census_id: str, offset: str) -> str: + """Resolve one bounded page from a durable raw-authority ledger.""" + try: + from polylogue.storage.raw_authority import read_raw_authority_census + + root = mcp_archive_root(hooks.get_config()) + handle = f"polylogue://raw-authority-census/{census_id}/{offset}" + return hooks.json_payload(MCPRootPayload(root=read_raw_authority_census(root, handle))) + except KeyError: + return hooks.error_json(f"Raw authority census not found: {census_id}", code="not_found") + except (FileNotFoundError, RuntimeError, ValueError) as exc: + return hooks.error_json( + f"Failed to read raw authority census {census_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + except Exception as exc: + return hooks.error_json( + f"Failed to read raw authority census {census_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + __all__ = ["register_resources"] diff --git a/polylogue/storage/archive_readiness.py b/polylogue/storage/archive_readiness.py index 77d722caff..52e2905101 100644 --- a/polylogue/storage/archive_readiness.py +++ b/polylogue/storage/archive_readiness.py @@ -251,7 +251,7 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "predecessor_census_id": census_row["predecessor_census_id"], "fixed_point": bool(census_row["fixed_point"]), "completed_at_ms": int(census_row["completed_at_ms"]), - "query_handle": f"raw-authority-census:{census_row['census_id']}", + "query_handle": (f"polylogue://raw-authority-census/{census_row['census_id']}/0"), } authority_blocker_count = 0 if _table_columns(conn, "source", "raw_authority_blockers"): diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 6105d9906a..a6a027531a 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -22,6 +22,7 @@ from polylogue.logging import get_logger RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v1" +RAW_AUTHORITY_CENSUS_QUERY_PREFIX = "polylogue://raw-authority-census/" logger = get_logger(__name__) @@ -94,7 +95,167 @@ class RawAuthorityCensusReceipt: @property def query_handle(self) -> str: - return f"raw-authority-census:{self.census_id}" + return raw_authority_census_query_handle(self.census_id) + + +def raw_authority_census_query_handle(census_id: str, *, offset: int = 0) -> str: + """Return a directly resolvable, paginated census-ledger URI.""" + if not census_id or "/" in census_id: + raise ValueError("raw authority census id must be non-empty and contain no slash") + if offset < 0: + raise ValueError("raw authority census offset must be non-negative") + return f"{RAW_AUTHORITY_CENSUS_QUERY_PREFIX}{census_id}/{offset}" + + +def _raw_authority_census_ref(value: str, *, offset: int | None) -> tuple[str, int]: + embedded_offset = 0 + if value.startswith(RAW_AUTHORITY_CENSUS_QUERY_PREFIX): + suffix = value.removeprefix(RAW_AUTHORITY_CENSUS_QUERY_PREFIX) + try: + census_id, encoded_offset = suffix.rsplit("/", 1) + embedded_offset = int(encoded_offset) + except (ValueError, TypeError) as exc: + raise ValueError("invalid raw authority census query handle") from exc + elif value.startswith("raw-authority-census:"): + # Read compatibility for the short-lived pre-URI receipt shape. + census_id = value.removeprefix("raw-authority-census:") + else: + census_id = value + resolved_offset = embedded_offset if offset is None else offset + if not census_id or "/" in census_id or resolved_offset < 0: + raise ValueError("invalid raw authority census query handle") + return census_id, resolved_offset + + +def _decode_json_field(value: object) -> object: + if not isinstance(value, str): + raise RuntimeError("raw authority ledger contains a non-text JSON field") + return json.loads(value) + + +def read_raw_authority_census( + archive_root: Path, + query_handle: str, + *, + limit: int = 100, + offset: int | None = None, +) -> JSONDocument: + """Read one bounded page from an immutable source-tier census ledger.""" + if not 1 <= limit <= 500: + raise ValueError("raw authority census limit must be between 1 and 500") + census_id, resolved_offset = _raw_authority_census_ref(query_handle, offset=offset) + source_db = archive_root / "source.db" + if not source_db.is_file(): + raise FileNotFoundError(source_db) + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + conn.row_factory = sqlite3.Row + census = conn.execute( + """ + SELECT census_id, sequence_no, scope_json, parser_fingerprint, + inventory_digest, residual_digest, plan_count, + executable_plan_count, residual_plan_count, + predecessor_census_id, fixed_point, created_at_ms, + completed_at_ms + FROM raw_authority_censuses WHERE census_id = ? + """, + (census_id,), + ).fetchone() + if census is None: + raise KeyError(census_id) + rows = conn.execute( + """ + SELECT cp.ordinal, cp.selected, cp.outcome_status, cp.reason, + cp.next_action, cp.application_receipt_json, cp.recorded_at_ms, + p.plan_id, p.input_digest, p.input_raw_ids_json, + p.logical_keys_json, p.authority_witness_json, + p.source_preconditions_json, p.index_preconditions_json, + p.created_at_ms + FROM raw_authority_census_plans AS cp + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE cp.census_id = ? + ORDER BY cp.ordinal + LIMIT ? OFFSET ? + """, + (census_id, limit, resolved_offset), + ).fetchall() + plan_ids = [str(row["plan_id"]) for row in rows] + blockers: list[dict[str, object]] = [] + if plan_ids: + marks = ",".join("?" for _ in plan_ids) + blockers = [ + { + "blocker_id": str(row["blocker_id"]), + "plan_id": str(row["plan_id"]), + "reason": str(row["reason"]), + "expected": _decode_json_field(row["expected_json"]), + "observed": _decode_json_field(row["observed_json"]), + "created_at_ms": int(row["created_at_ms"]), + "resolved_at_ms": row["resolved_at_ms"], + "resolution": row["resolution"], + } + for row in conn.execute( + f""" + SELECT blocker_id, plan_id, reason, expected_json, + observed_json, created_at_ms, resolved_at_ms, resolution + FROM raw_authority_blockers + WHERE plan_id IN ({marks}) + ORDER BY created_at_ms, blocker_id + """, + plan_ids, + ) + ] + plans = [ + { + "ordinal": int(row["ordinal"]), + "selected": bool(row["selected"]), + "outcome_status": str(row["outcome_status"]), + "reason": str(row["reason"]), + "next_action": str(row["next_action"]), + "application_receipt": _decode_json_field(row["application_receipt_json"]), + "recorded_at_ms": int(row["recorded_at_ms"]), + "plan": { + "plan_id": str(row["plan_id"]), + "input_digest": str(row["input_digest"]), + "input_raw_ids": _decode_json_field(row["input_raw_ids_json"]), + "logical_keys": _decode_json_field(row["logical_keys_json"]), + "authority_witness": _decode_json_field(row["authority_witness_json"]), + "source_preconditions": _decode_json_field(row["source_preconditions_json"]), + "index_preconditions": _decode_json_field(row["index_preconditions_json"]), + "created_at_ms": int(row["created_at_ms"]), + }, + } + for row in rows + ] + total = int(census["plan_count"]) + next_offset = resolved_offset + len(plans) + return json_document( + { + "query_handle": raw_authority_census_query_handle(census_id, offset=resolved_offset), + "next_query_handle": ( + raw_authority_census_query_handle(census_id, offset=next_offset) if next_offset < total else None + ), + "offset": resolved_offset, + "limit": limit, + "returned_count": len(plans), + "census": { + "census_id": str(census["census_id"]), + "sequence_no": int(census["sequence_no"]), + "scope": _decode_json_field(census["scope_json"]), + "parser_fingerprint": str(census["parser_fingerprint"]), + "inventory_digest": str(census["inventory_digest"]), + "residual_digest": str(census["residual_digest"]), + "plan_count": total, + "executable_plan_count": int(census["executable_plan_count"]), + "residual_plan_count": int(census["residual_plan_count"]), + "predecessor_census_id": census["predecessor_census_id"], + "fixed_point": bool(census["fixed_point"]), + "created_at_ms": int(census["created_at_ms"]), + "completed_at_ms": int(census["completed_at_ms"]), + }, + "plans": plans, + "blockers": blockers, + } + ) def _canonical_json(value: object) -> str: @@ -273,6 +434,7 @@ def record_raw_authority_census( plans: Sequence[RawReplayPlan], *, selected_plan_ids: set[str], + executable_plan_ids: set[str] | None = None, scope: Mapping[str, object], residual: Mapping[str, object], ) -> RawAuthorityCensusReceipt: @@ -291,7 +453,12 @@ def record_raw_authority_census( ).fetchone() sequence_no = int(previous[1]) + 1 if previous is not None else 1 predecessor = str(previous[0]) if previous is not None else None - executable_count = len(selected_plan_ids) + executable_ids = selected_plan_ids if executable_plan_ids is None else executable_plan_ids + unknown_ids = (selected_plan_ids | executable_ids) - {plan.plan_id for plan in plans} + if unknown_ids: + raise RuntimeError(f"raw authority census references unknown plans: {sorted(unknown_ids)}") + executable_count = len(executable_ids) + residual_count = len(plans) - len(selected_plan_ids) fixed_point = bool( previous is not None and int(previous[4]) == 0 @@ -352,7 +519,7 @@ def record_raw_authority_census( residual_digest, len(plans), executable_count, - len(plans) - executable_count, + residual_count, predecessor, int(fixed_point), now, @@ -387,7 +554,7 @@ def record_raw_authority_census( residual_digest=residual_digest, plan_count=len(plans), executable_plan_count=executable_count, - residual_plan_count=len(plans) - executable_count, + residual_plan_count=residual_count, predecessor_census_id=predecessor, fixed_point=fixed_point, ) @@ -530,6 +697,7 @@ def reject_stale_raw_replay_plan( __all__ = [ + "RAW_AUTHORITY_CENSUS_QUERY_PREFIX", "RAW_AUTHORITY_PARSER_FINGERPRINT", "RawAuthorityCensusReceipt", "RawReplayPlan", @@ -538,7 +706,9 @@ def reject_stale_raw_replay_plan( "build_raw_replay_plan", "build_raw_replay_plans", "raw_replay_application_receipt", + "raw_authority_census_query_handle", "raw_replay_plan_last_attempts", + "read_raw_authority_census", "record_raw_authority_census", "record_raw_replay_outcome", "reject_stale_raw_replay_plan", diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 4ccccc01cb..26fb2b4dcd 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -6498,32 +6498,32 @@ def repair_raw_materialization( metrics["raw_materialization_execute_blob_limit_bytes"] = float(RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES) if oversized_stream_safe_raw_ids: metrics["raw_materialization_stream_oversized_count"] = float(len(oversized_stream_safe_raw_ids)) - census_receipt: RawAuthorityCensusReceipt | None = None - if not dry_run: - census_receipt = record_raw_authority_census( - archive_root, - plans, - selected_plan_ids={plan_by_component[component].plan_id for component in selected_components}, - scope={ - "raw_artifact_id": raw_artifact_id, - "provider": provider, - "source_family": source_family, - "source_root": str(source_root) if source_root is not None else None, - "raw_artifact_limit": raw_artifact_limit, - }, - residual={ - "missing_blobs": missing_blobs, - "missing_blob_source_available": candidates.missing_blob_source_available, - "missing_blob_source_missing": candidates.missing_blob_source_missing, - "adoption_deferred": candidates.adoption_deferred, - "authority_quarantined": candidates.authority_quarantined, - "byte_authority_fragments": candidates.byte_authority_fragments, - "byte_authority_quarantined": candidates.byte_authority_quarantined, - "byte_authority_pending": candidates.byte_authority_pending, - }, - ) - metrics["raw_materialization_census_sequence"] = float(census_receipt.sequence_no) - metrics["raw_materialization_census_fixed_point"] = float(census_receipt.fixed_point) + selected_plan_ids = {plan_by_component[component].plan_id for component in selected_components} + census_receipt = record_raw_authority_census( + archive_root, + plans, + selected_plan_ids=set() if dry_run else selected_plan_ids, + executable_plan_ids={plan.plan_id for plan in plans}, + scope={ + "raw_artifact_id": raw_artifact_id, + "provider": provider, + "source_family": source_family, + "source_root": str(source_root) if source_root is not None else None, + "raw_artifact_limit": raw_artifact_limit, + }, + residual={ + "missing_blobs": missing_blobs, + "missing_blob_source_available": candidates.missing_blob_source_available, + "missing_blob_source_missing": candidates.missing_blob_source_missing, + "adoption_deferred": candidates.adoption_deferred, + "authority_quarantined": candidates.authority_quarantined, + "byte_authority_fragments": candidates.byte_authority_fragments, + "byte_authority_quarantined": candidates.byte_authority_quarantined, + "byte_authority_pending": candidates.byte_authority_pending, + }, + ) + metrics["raw_materialization_census_sequence"] = float(census_receipt.sequence_no) + metrics["raw_materialization_census_fixed_point"] = float(census_receipt.fixed_point) if not candidate_raw_ids: detail = "Executable raw replay converged" if ( @@ -6606,8 +6606,6 @@ def repair_raw_materialization( census_receipt=census_receipt, ) - if census_receipt is None: - raise RuntimeError("apply-mode raw materialization requires a durable census receipt") for outcome in blocked_plan_outcomes: record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) diff --git a/tests/infra/mcp.py b/tests/infra/mcp.py index 223a8f3287..dca0068644 100644 --- a/tests/infra/mcp.py +++ b/tests/infra/mcp.py @@ -128,6 +128,7 @@ EXPECTED_RESOURCE_TEMPLATE_URIS = { "polylogue://session/{conv_id}", + "polylogue://raw-authority-census/{census_id}/{offset}", } EXPECTED_PROMPT_NAMES = { diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index 642ac09d9a..286e82e7b7 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -17,6 +17,7 @@ from polylogue.sources.live.cursor import CursorStore from polylogue.storage.blob_gc import read_gc_history from polylogue.storage.blob_publication import ArchiveBlobPublisher +from polylogue.storage.raw_authority import record_raw_authority_census from polylogue.storage.sqlite.archive_tiers.archive import ArchiveSessionSearchHit, ArchiveSessionSummary from polylogue.storage.sqlite.archive_tiers.archive_init import ( ArchiveInitResult, @@ -32,6 +33,40 @@ _ARCHIVE_TIERS = ("source.db", "index.db", "embeddings.db", "ops.db", "user.db") +def test_raw_authority_census_cli_resolves_receipt_handle( + cli_workspace: dict[str, Path], + cli_runner: CliRunner, +) -> None: + root = cli_workspace["archive_root"] + receipt = record_raw_authority_census( + root, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + scope={"source_family": "codex"}, + residual={}, + ) + + result = cli_runner.invoke( + cli, + [ + "--plain", + "ops", + "maintenance", + "raw-authority-census", + receipt.query_handle, + "--output-format", + "json", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["query_handle"] == receipt.query_handle + assert payload["census"]["census_id"] == receipt.census_id + + def _stage_uninitialized_archive(cli_workspace: dict[str, Path]) -> None: """Reset the workspace to an uninitialized state for plan/init tests. diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index c21f7ca8f5..4e9ccba7cf 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -614,7 +614,9 @@ def test_raw_materialization_pass_emits_conserved_plan_receipt(monkeypatch: pyte "raw_materialization_candidate_count": 1.0, "raw_materialization_remaining_candidate_count": 0.0, }, - "plan_outcomes": [outcome.to_dict()], + "plan_outcome_count": 1, + "plan_outcome_sample": [outcome.to_dict()], + "plan_outcome_sample_truncated": False, }, ) ] @@ -641,7 +643,34 @@ def test_raw_materialization_pass_emits_zero_work_receipt(monkeypatch: pytest.Mo ) assert events[0][1]["success"] is True - assert events[0][1]["plan_outcomes"] == [] + assert events[0][1]["plan_outcome_count"] == 0 + assert events[0][1]["plan_outcome_sample"] == [] + assert events[0][1]["plan_outcome_sample_truncated"] is False + + +def test_raw_materialization_pass_bounds_outcome_sample(monkeypatch: pytest.MonkeyPatch) -> None: + from polylogue.daemon import cli as daemon_cli + + events: list[tuple[str, dict[str, object]]] = [] + outcomes = tuple(SimpleNamespace(to_dict=lambda index=index: {"plan_id": f"plan:{index}"}) for index in range(9)) + monkeypatch.setattr( + "polylogue.daemon.events.emit_daemon_event", + lambda kind, *, payload: events.append((kind, payload)), + ) + + daemon_cli._emit_raw_materialization_pass( + SimpleNamespace( + success=True, + repaired_count=0, + detail="bounded", + metrics={}, + plan_outcomes=outcomes, + ) + ) + + assert events[0][1]["plan_outcome_count"] == 9 + assert len(cast(list[object], events[0][1]["plan_outcome_sample"])) == 8 + assert events[0][1]["plan_outcome_sample_truncated"] is True def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: pytest.MonkeyPatch) -> None: @@ -663,7 +692,7 @@ def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: py residual_plan_count=2, predecessor_census_id="census:1:inventory:residual", fixed_point=False, - query_handle="raw-authority-census:census:2:inventory:residual", + query_handle="polylogue://raw-authority-census/census:2:inventory:residual/0", ) daemon_cli._emit_raw_materialization_pass( diff --git a/tests/unit/mcp/test_server_surfaces.py b/tests/unit/mcp/test_server_surfaces.py index 4c5319a800..4af4be7897 100644 --- a/tests/unit/mcp/test_server_surfaces.py +++ b/tests/unit/mcp/test_server_surfaces.py @@ -20,6 +20,7 @@ from polylogue.core.refs import EvidenceRef from polylogue.core.types import SessionId from polylogue.sources.parsers.base import ParsedContentBlock, ParsedMessage, ParsedSession +from polylogue.storage.raw_authority import record_raw_authority_census from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.user_write import ArchiveAssertionEnvelope from polylogue.surfaces.payloads import ( @@ -558,6 +559,36 @@ def test_readiness_resource(self: object, mcp_server: MCPServerUnderTest) -> Non assert component_readiness["index"]["state"] == "degraded" assert component_readiness["index"]["caveats"] == ["messages_fts_row_mismatch"] + def test_raw_authority_census_query_handle_resolves_bounded_ledger( + self: object, mcp_server: MCPServerUnderTest, tmp_path: Path + ) -> None: + archive_root = tmp_path / "archive" + with ArchiveStore(archive_root): + pass + receipt = record_raw_authority_census( + archive_root, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + scope={"origin": "codex-session"}, + residual={}, + ) + with patch("polylogue.mcp.server._get_config") as mock_get_config: + mock_get_config.return_value = SimpleNamespace( + archive_root=archive_root, + db_path=archive_root / "index.db", + ) + result = invoke_surface( + mcp_server._resource_manager._templates["polylogue://raw-authority-census/{census_id}/{offset}"].fn, + census_id=receipt.census_id, + offset="0", + ) + + payload = json.loads(result) + assert payload["query_handle"] == receipt.query_handle + assert payload["census"]["census_id"] == receipt.census_id + assert payload["plans"] == [] + class TestArchiveGenericToolSurfaces: @pytest.mark.asyncio diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index a673d1ea32..155a4e2d7b 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -12,6 +12,7 @@ from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot from polylogue.storage.raw_authority import ( build_raw_replay_plans, + read_raw_authority_census, record_raw_authority_census, reject_stale_raw_replay_plan, validate_raw_replay_plan, @@ -86,7 +87,7 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat assert result.census_receipt is not None assert result.census_receipt.plan_count == 2 - assert result.census_receipt.executable_plan_count == 1 + assert result.census_receipt.executable_plan_count == 2 assert result.census_receipt.residual_plan_count == 1 assert result.metrics["raw_materialization_plan_outcome_count"] == 2.0 assert result.metrics["raw_materialization_plan_carried_forward_count"] == 1.0 @@ -124,9 +125,19 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat assert census_status["inventory_digest"] == result.census_receipt.inventory_digest assert census_status["residual_digest"] == result.census_receipt.residual_digest assert census_status["plan_count"] == 2 - assert census_status["executable_plan_count"] == 1 + assert census_status["executable_plan_count"] == 2 assert census_status["residual_plan_count"] == 1 assert census_status["query_handle"] == result.census_receipt.query_handle + first_page = read_raw_authority_census(tmp_path, result.census_receipt.query_handle, limit=1) + assert first_page["returned_count"] == 1 + assert first_page["next_query_handle"] is not None + second_page = read_raw_authority_census(tmp_path, cast(str, first_page["next_query_handle"]), limit=1) + assert second_page["returned_count"] == 1 + assert second_page["next_query_handle"] is None + assert { + cast(dict[str, object], item)["outcome_status"] + for item in (*cast(list[object], first_page["plans"]), *cast(list[object], second_page["plans"])) + } == {"executed", "carried_forward"} def test_two_successive_quiescent_censuses_are_required_for_fixed_point(tmp_path: Path) -> None: @@ -134,8 +145,8 @@ def test_two_successive_quiescent_censuses_are_required_for_fixed_point(tmp_path _write_codex_raw(tmp_path, native_id="fixed", source_path="fixed.jsonl", acquired_at_ms=1) assert repair_raw_materialization(_config(tmp_path)).repaired_count == 1 - first_empty = repair_raw_materialization(_config(tmp_path)) - second_empty = repair_raw_materialization(_config(tmp_path)) + first_empty = repair_raw_materialization(_config(tmp_path), dry_run=True) + second_empty = repair_raw_materialization(_config(tmp_path), dry_run=True) assert first_empty.census_receipt is not None assert second_empty.census_receipt is not None diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 6e60a01e1b..718b350c8b 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -256,6 +256,8 @@ def test_raw_materialization_preview_counts_replayable_rows_without_erasing_miss "raw_materialization_before_component_count": 1.0, "raw_materialization_selected_executable_component_count": 1.0, "raw_materialization_selected_blocked_component_count": 0.0, + "raw_materialization_census_sequence": 1.0, + "raw_materialization_census_fixed_point": 0.0, } assert "per-session revision authority" in result.detail assert "selected raw payload bytes total=" in result.detail From ee896262369baef8c843deefce752ae904619065 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 16 Jul 2026 23:24:50 +0200 Subject: [PATCH 05/13] fix(storage): make raw authority plans crash-safe Problem: immutable replay plans could be published before every relevant raw had current parser evidence, and apply censuses were marked complete before exact outcomes were durable. Parsed timestamps could therefore masquerade as successful application and stale blockers had no supported resolution path. What changed: persist per-raw parser census receipts, gate planning on quiescence, distinguish census/dry-run/apply lifecycle, recover interrupted applications from exact source/index postconditions, fail closed on invalid receipts, record identity-sensitive residual debt, expose bounded lifecycle projections, and add an explicit receipted blocker resolver. Ref polylogue-hjpx.1. Ref polylogue-lkrc. Co-Authored-By: Claude --- docs/maintenance.md | 23 + .../cli/commands/maintenance/__init__.py | 6 + .../cli/commands/maintenance/_raw_identity.py | 38 ++ polylogue/daemon/cli.py | 3 + polylogue/sources/revision_backfill.py | 97 ++++ polylogue/storage/archive_readiness.py | 24 +- polylogue/storage/raw_authority.py | 431 +++++++++++++++++- polylogue/storage/repair.py | 320 ++++++++++--- .../storage/sqlite/archive_tiers/source.py | 22 +- .../source/013_raw_authority_ledger.sql | 22 +- .../unit/cli/test_archive_maintenance_cli.py | 33 ++ tests/unit/daemon/test_daemon_cli.py | 6 + tests/unit/mcp/test_server_surfaces.py | 2 + tests/unit/storage/test_durable_migrations.py | 2 + .../unit/storage/test_raw_authority_ledger.py | 261 ++++++++++- tests/unit/storage/test_repair.py | 10 +- 16 files changed, 1200 insertions(+), 100 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index 556ae27eba..327340887c 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -353,6 +353,29 @@ more rows remain. `--limit` is bounded to 1–500; `--offset` can override the offset encoded in the URI. MCP clients resolve the URI directly through the matching resource template. +Every receipt identifies its `mode` (`census`, `dry_run`, or `apply`), whether +the parser census was `quiescent`, and its lifecycle. Apply receipts remain +`planned` until every selected immutable plan has an outcome; startup recovery +then validates exact source, application/membership, accepted-head, and session +postconditions before marking an interrupted pass `executed`. Readiness never +reports a `planned` row as the latest completed census and exposes its pending +count separately. + +A stale precondition or incomplete application receipt creates a durable, +fail-closed blocker. After inspecting the census URI and current evidence, +explicitly reopen replanning with a recorded rationale: + +```bash +polylogue ops maintenance raw-authority-blocker-resolve \ + --blocker-id 'raw-authority-blocker:...' \ + --reason 'reviewed current source/index evidence; replan from this state' \ + --yes +``` + +Resolution never applies the stale plan. It stores the replacement plan +witness in the resolution receipt; the next ordinary convergence pass plans +and validates current evidence normally. + ### Draining the convergence-debt queue **Symptoms.** `polylogue ops diagnostics workload` reports a non-trivial diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 304436a1b1..847167a666 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -59,6 +59,12 @@ "raw_authority_census_command", "Read a bounded page from a durable raw-authority census ledger.", ), + ( + "raw-authority-blocker-resolve", + "_raw_identity", + "raw_authority_blocker_resolve_command", + "Resolve one stale-plan blocker against current source evidence.", + ), ( "missing-raw-blob-cursors", "_raw_identity", diff --git a/polylogue/cli/commands/maintenance/_raw_identity.py b/polylogue/cli/commands/maintenance/_raw_identity.py index 6779ecf2ec..c3e354cbef 100644 --- a/polylogue/cli/commands/maintenance/_raw_identity.py +++ b/polylogue/cli/commands/maintenance/_raw_identity.py @@ -65,6 +65,44 @@ def raw_authority_census_command( click.echo(f"Next: {next_handle}") +@click.command("raw-authority-blocker-resolve") +@click.option("--blocker-id", required=True, help="Exact unresolved durable blocker identifier.") +@click.option("--reason", required=True, help="Operator rationale recorded in the immutable resolution receipt.") +@click.option("--yes", "confirmed", is_flag=True, help="Confirm resolving this blocker against current evidence.") +@click.option( + "--output-format", + "output_format", + type=click.Choice(["plain", "json"]), + default="plain", + show_default=True, +) +@click.pass_obj +def raw_authority_blocker_resolve_command( + env: AppEnv, + blocker_id: str, + reason: str, + confirmed: bool, + output_format: str, +) -> None: + """Resolve one stale-plan blocker after replanning current evidence.""" + del env + if not confirmed: + raise click.ClickException("refusing to resolve a durable blocker without --yes") + from polylogue.storage.raw_authority import resolve_raw_authority_blocker + + try: + receipt = resolve_raw_authority_blocker(archive_root(), blocker_id, resolution=reason) + except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(receipt, indent=2, sort_keys=True)) + return + click.echo(f"Resolved {blocker_id}") + current_plan = receipt.get("current_plan") + if isinstance(current_plan, dict): + click.echo(f"Current plan: {current_plan.get('plan_id', 'unknown')}") + + def _raw_blob_path_for_hash(root: Path, blob_hash: bytes | str) -> Path | None: hex_hash = blob_hash.hex() if isinstance(blob_hash, bytes) else str(blob_hash).lower() if len(hex_hash) != 64 or any(char not in "0123456789abcdef" for char in hex_hash): diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index d96c755482..00b835b998 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -655,6 +655,9 @@ def _emit_raw_materialization_pass(result: Any) -> None: "executable_plan_count": census.executable_plan_count, "residual_plan_count": census.residual_plan_count, "predecessor_census_id": census.predecessor_census_id, + "mode": census.mode, + "lifecycle_status": census.lifecycle_status, + "quiescent": census.quiescent, "fixed_point": census.fixed_point, "query_handle": census.query_handle, } diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 6da31989b5..1bc876a7ac 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import pickle import sqlite3 @@ -63,6 +64,101 @@ def __init__(self, raw_ids: list[str], limit_bytes: int, total_bytes: int) -> No super().__init__(f"{len(raw_ids)} raw revision(s) total {total_bytes} bytes exceed replay limit {limit_bytes}") +def uncensused_historical_revision_raw_ids(archive_root: Path, raw_ids: list[str]) -> tuple[str, ...]: + """Return inputs whose current parser identity has not been persisted. + + The dedicated receipt proves that the current parser actually observed + every relevant raw. Durable revision or membership rows alone may have + been produced by an older parser and therefore cannot establish current + quiescence. + """ + if not raw_ids: + return () + placeholders = ",".join("?" for _ in raw_ids) + with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as conn: + rows = conn.execute( + f""" + SELECT r.raw_id + FROM raw_sessions AS r + LEFT JOIN raw_authority_parser_census AS c ON c.raw_id = r.raw_id + WHERE r.raw_id IN ({placeholders}) + AND NOT COALESCE( + c.parser_fingerprint = 'revision-membership-v1' + AND c.status = 'complete', + 0 + ) + ORDER BY r.raw_id + """, + raw_ids, + ).fetchall() + return tuple(str(row[0]) for row in rows) + + +def _record_raw_authority_parser_census(archive_root: Path, raw_ids: tuple[str, ...]) -> None: + """Persist per-raw current-parser completion without changing governance.""" + if not raw_ids: + return + with sqlite3.connect(archive_root / "source.db") as conn, conn: + for raw_id in raw_ids: + raw = conn.execute( + """ + SELECT logical_source_key, revision_kind + FROM raw_sessions WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() + membership_census = conn.execute( + """ + SELECT status, detail FROM raw_membership_census + WHERE raw_id = ? AND parser_fingerprint = 'revision-membership-v1' + """, + (raw_id,), + ).fetchone() + membership_keys = [ + str(row[0]) + for row in conn.execute( + """ + SELECT logical_source_key FROM raw_session_memberships + WHERE raw_id = ? ORDER BY logical_source_key + """, + (raw_id,), + ) + ] + typed_key = ( + str(raw[0]) + if raw is not None and raw[0] is not None and str(raw[1]) != RawRevisionKind.UNKNOWN.value + else None + ) + complete = typed_key is not None or ( + membership_census is not None and str(membership_census[0]) in {"complete", "non_session"} + ) + logical_keys = sorted(set(membership_keys) | ({typed_key} if typed_key is not None else set())) + detail = ( + "current parser established durable authority identity" + if complete + else ( + str(membership_census[1]) + if membership_census is not None + else "current parser produced no durable authority identity" + ) + ) + conn.execute( + """ + INSERT INTO raw_authority_parser_census ( + raw_id, parser_fingerprint, status, logical_keys_json, + detail, censused_at_ms + ) VALUES (?, 'revision-membership-v1', ?, ?, ?, 0) + ON CONFLICT(raw_id) DO UPDATE SET + parser_fingerprint = excluded.parser_fingerprint, + status = excluded.status, + logical_keys_json = excluded.logical_keys_json, + detail = excluded.detail, + censused_at_ms = excluded.censused_at_ms + """, + (raw_id, "complete" if complete else "failed", json.dumps(logical_keys), detail), + ) + + def _census_historical_revision_evidence( archive: ArchiveStore, spill: _ParsedSessionSpill, @@ -169,6 +265,7 @@ def census_historical_revision_evidence( max_payload_bytes=max_payload_bytes, ) expanded, logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) + _record_raw_authority_parser_census(archive_root, tuple(expanded)) return RevisionCensusResult( state.scanned, state.classified, diff --git a/polylogue/storage/archive_readiness.py b/polylogue/storage/archive_readiness.py index 52e2905101..3ec01f675f 100644 --- a/polylogue/storage/archive_readiness.py +++ b/polylogue/storage/archive_readiness.py @@ -229,13 +229,24 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob lost_source_evidence_count = _missing_source_raw_session_count(conn) lost_source_evidence_samples = _missing_source_raw_session_samples(conn) authority_census: dict[str, object] | None = None + authority_pending_census_count = 0 if _table_columns(conn, "source", "raw_authority_censuses"): + authority_pending_census_count = int( + conn.execute( + """ + SELECT COUNT(*) FROM source.raw_authority_censuses + WHERE lifecycle_status = 'planned' + """ + ).fetchone()[0] + ) census_row = conn.execute( """ SELECT census_id, sequence_no, inventory_digest, residual_digest, plan_count, executable_plan_count, residual_plan_count, - predecessor_census_id, fixed_point, completed_at_ms + predecessor_census_id, mode, lifecycle_status, quiescent, + fixed_point, completed_at_ms FROM source.raw_authority_censuses + WHERE lifecycle_status IN ('completed', 'interrupted') ORDER BY sequence_no DESC LIMIT 1 """ ).fetchone() @@ -249,8 +260,12 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "executable_plan_count": int(census_row["executable_plan_count"]), "residual_plan_count": int(census_row["residual_plan_count"]), "predecessor_census_id": census_row["predecessor_census_id"], + "mode": str(census_row["mode"]), + "lifecycle_status": str(census_row["lifecycle_status"]), + "quiescent": bool(census_row["quiescent"]), "fixed_point": bool(census_row["fixed_point"]), "completed_at_ms": int(census_row["completed_at_ms"]), + "pending_census_count": authority_pending_census_count, "query_handle": (f"polylogue://raw-authority-census/{census_row['census_id']}/0"), } authority_blocker_count = 0 @@ -292,6 +307,8 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob category_counts["adoption_deferred"] = adoption_deferred_count if authority_blocker_count: category_counts["raw_authority_blocker"] = authority_blocker_count + if authority_pending_census_count: + category_counts["raw_authority_pending_census"] = authority_pending_census_count category_counts.update( {category: count for category, count in classified_counts.items() if category != "parse-failed"} ) @@ -307,12 +324,12 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "critical": critical, "warning": 0, "actionable": actionable, - "blocked": adoption_deferred_count + authority_blocker_count, + "blocked": adoption_deferred_count + authority_blocker_count + authority_pending_census_count, "classified": classified, "unchecked": unchecked, "affected_total": total, "affected_actionable": affected_actionable, - "affected_blocked": adoption_deferred_count + authority_blocker_count, + "affected_blocked": adoption_deferred_count + authority_blocker_count + authority_pending_census_count, "affected_open": 0, "affected_classified": classified, "affected_unchecked": unchecked, @@ -322,6 +339,7 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "source_family_counts": {str(item["origin"]): int(item["count"] or 0) for item in family_rows}, "raw_authority_census": authority_census, "raw_authority_blocker_count": authority_blocker_count, + "raw_authority_pending_census_count": authority_pending_census_count, } diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index a6a027531a..862eea0d64 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -91,6 +91,9 @@ class RawAuthorityCensusReceipt: executable_plan_count: int residual_plan_count: int predecessor_census_id: str | None + mode: str + lifecycle_status: str + quiescent: bool fixed_point: bool @property @@ -151,7 +154,8 @@ def read_raw_authority_census( conn.row_factory = sqlite3.Row census = conn.execute( """ - SELECT census_id, sequence_no, scope_json, parser_fingerprint, + SELECT census_id, sequence_no, scope_json, residual_json, parser_fingerprint, + mode, lifecycle_status, quiescent, inventory_digest, residual_digest, plan_count, executable_plan_count, residual_plan_count, predecessor_census_id, fixed_point, created_at_ms, @@ -166,6 +170,7 @@ def read_raw_authority_census( """ SELECT cp.ordinal, cp.selected, cp.outcome_status, cp.reason, cp.next_action, cp.application_receipt_json, cp.recorded_at_ms, + cp.outcome_recorded, p.plan_id, p.input_digest, p.input_raw_ids_json, p.logical_keys_json, p.authority_witness_json, p.source_preconditions_json, p.index_preconditions_json, @@ -212,6 +217,7 @@ def read_raw_authority_census( "reason": str(row["reason"]), "next_action": str(row["next_action"]), "application_receipt": _decode_json_field(row["application_receipt_json"]), + "outcome_recorded": bool(row["outcome_recorded"]), "recorded_at_ms": int(row["recorded_at_ms"]), "plan": { "plan_id": str(row["plan_id"]), @@ -241,7 +247,11 @@ def read_raw_authority_census( "census_id": str(census["census_id"]), "sequence_no": int(census["sequence_no"]), "scope": _decode_json_field(census["scope_json"]), + "residual": _decode_json_field(census["residual_json"]), "parser_fingerprint": str(census["parser_fingerprint"]), + "mode": str(census["mode"]), + "lifecycle_status": str(census["lifecycle_status"]), + "quiescent": bool(census["quiescent"]), "inventory_digest": str(census["inventory_digest"]), "residual_digest": str(census["residual_digest"]), "plan_count": total, @@ -250,7 +260,7 @@ def read_raw_authority_census( "predecessor_census_id": census["predecessor_census_id"], "fixed_point": bool(census["fixed_point"]), "created_at_ms": int(census["created_at_ms"]), - "completed_at_ms": int(census["completed_at_ms"]), + "completed_at_ms": (int(census["completed_at_ms"]) if census["completed_at_ms"] is not None else None), }, "plans": plans, "blockers": blockers, @@ -320,6 +330,15 @@ def build_raw_replay_plan(conn: sqlite3.Connection, input_raw_ids: Sequence[str] """, raw_ids, ) + parser_census_rows = _rows( + conn, + f""" + SELECT raw_id, parser_fingerprint, status, logical_keys_json, detail + FROM raw_authority_parser_census + WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + raw_ids, + ) logical_keys = tuple( sorted( { @@ -356,12 +375,15 @@ def build_raw_replay_plan(conn: sqlite3.Connection, input_raw_ids: Sequence[str] ) authority_witness = json_document( { + "parser_census": parser_census_rows, "membership_census": census_rows, "memberships": membership_rows, "revision_heads": head_rows, } ) - source_preconditions = json_document({"raw_sessions": source_rows}) + source_preconditions = json_document( + {"raw_sessions": source_rows, "raw_authority_parser_census": parser_census_rows} + ) index_preconditions = json_document({"sessions": session_rows, "revision_heads": head_rows}) identity = { "schema": "polylogue.raw-replay-plan.v2", @@ -406,9 +428,12 @@ def raw_replay_plan_last_attempts(archive_root: Path) -> dict[str, int]: str(row[0]): int(row[1]) for row in conn.execute( """ - SELECT plan_id, MAX(recorded_at_ms) - FROM raw_authority_census_plans - WHERE selected = 1 GROUP BY plan_id + SELECT cp.plan_id, MAX(cp.recorded_at_ms) + FROM raw_authority_census_plans AS cp + JOIN raw_authority_censuses AS c ON c.census_id = cp.census_id + WHERE cp.selected = 1 + AND c.lifecycle_status IN ('completed', 'interrupted') + GROUP BY cp.plan_id """ ) } @@ -435,19 +460,27 @@ def record_raw_authority_census( *, selected_plan_ids: set[str], executable_plan_ids: set[str] | None = None, + mode: str, + quiescent: bool, scope: Mapping[str, object], residual: Mapping[str, object], ) -> RawAuthorityCensusReceipt: - """Atomically publish a complete plan census with carried-forward outcomes.""" + """Atomically publish a plan census, finalized only when no apply is pending.""" + if mode not in {"census", "dry_run", "apply"}: + raise ValueError(f"unsupported raw authority census mode: {mode}") + if mode != "apply" and selected_plan_ids: + raise ValueError(f"{mode} census cannot select plans for application") now = int(time.time() * 1000) inventory_digest = _digest([plan.plan_id for plan in plans]) residual_digest = _digest(residual) scope_json = _canonical_json(scope) + residual_json = _canonical_json(residual) with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: previous = conn.execute( """ SELECT census_id, sequence_no, inventory_digest, residual_digest, - executable_plan_count, scope_json + executable_plan_count, scope_json, parser_fingerprint, + mode, lifecycle_status, quiescent FROM raw_authority_censuses ORDER BY sequence_no DESC LIMIT 1 """ ).fetchone() @@ -458,7 +491,7 @@ def record_raw_authority_census( if unknown_ids: raise RuntimeError(f"raw authority census references unknown plans: {sorted(unknown_ids)}") executable_count = len(executable_ids) - residual_count = len(plans) - len(selected_plan_ids) + residual_count = len(plans) - executable_count fixed_point = bool( previous is not None and int(previous[4]) == 0 @@ -466,8 +499,16 @@ def record_raw_authority_census( and str(previous[2]) == inventory_digest and str(previous[3]) == residual_digest and str(previous[5]) == scope_json + and str(previous[6]) == RAW_AUTHORITY_PARSER_FINGERPRINT + and str(previous[7]) == "dry_run" + and str(previous[8]) == "completed" + and bool(previous[9]) + and mode == "dry_run" + and quiescent ) census_id = f"census:{sequence_no}:{inventory_digest[:16]}:{residual_digest[:16]}" + lifecycle_status = "planned" if mode == "apply" and selected_plan_ids else "completed" + completed_at_ms = None if lifecycle_status == "planned" else now for plan in plans: values = ( plan.plan_id, @@ -504,17 +545,22 @@ def record_raw_authority_census( conn.execute( """ INSERT INTO raw_authority_censuses ( - census_id, sequence_no, scope_json, parser_fingerprint, + census_id, sequence_no, scope_json, residual_json, + parser_fingerprint, mode, lifecycle_status, quiescent, inventory_digest, residual_digest, plan_count, executable_plan_count, residual_plan_count, predecessor_census_id, fixed_point, created_at_ms, completed_at_ms - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( census_id, sequence_no, scope_json, + residual_json, RAW_AUTHORITY_PARSER_FINGERPRINT, + mode, + lifecycle_status, + int(quiescent), inventory_digest, residual_digest, len(plans), @@ -523,7 +569,7 @@ def record_raw_authority_census( predecessor, int(fixed_point), now, - now, + completed_at_ms, ), ) for ordinal, plan in enumerate(plans): @@ -532,18 +578,21 @@ def record_raw_authority_census( """ INSERT INTO raw_authority_census_plans ( census_id, plan_id, ordinal, selected, outcome_status, - reason, next_action, application_receipt_json, recorded_at_ms - ) VALUES (?, ?, ?, ?, 'carried_forward', ?, ?, '{}', ?) + reason, next_action, application_receipt_json, + outcome_recorded, recorded_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, '{}', ?, ?) """, ( census_id, plan.plan_id, ordinal, int(selected), - "selected plan awaits a typed application outcome" + RawReplayPlanStatus.RETRYABLE.value if selected else RawReplayPlanStatus.CARRIED_FORWARD.value, + "selected plan application is pending" if selected else "bounded scheduler carried this complete plan forward unchanged", "execute this plan in the current pass" if selected else "retain for a later bounded pass", + 0 if selected else 1, now, ), ) @@ -556,6 +605,9 @@ def record_raw_authority_census( executable_plan_count=executable_count, residual_plan_count=residual_count, predecessor_census_id=predecessor, + mode=mode, + lifecycle_status=lifecycle_status, + quiescent=quiescent, fixed_point=fixed_point, ) @@ -601,16 +653,106 @@ def raw_replay_application_receipt(archive_root: Path, plan: RawReplayPlan) -> J """, plan.input_raw_ids, ) + if plan.logical_keys: + key_marks = ",".join("?" for _ in plan.logical_keys) + heads = _rows( + conn, + f""" + SELECT logical_source_key, session_id, accepted_raw_id, + accepted_source_revision, + hex(accepted_content_hash) AS accepted_content_hash, + accepted_frontier_kind, accepted_frontier + FROM index_tier.raw_revision_heads + WHERE logical_source_key IN ({key_marks}) + ORDER BY logical_source_key + """, + plan.logical_keys, + ) + sessions = _rows( + conn, + f""" + SELECT s.session_id, s.raw_id, hex(s.content_hash) AS content_hash, + s.message_count + FROM index_tier.sessions AS s + JOIN index_tier.raw_revision_heads AS h ON h.session_id = s.session_id + WHERE h.logical_source_key IN ({key_marks}) + ORDER BY s.session_id + """, + plan.logical_keys, + ) + else: + heads = [] + sessions = [] return json_document( { - "schema": "polylogue.raw-replay-application-receipt.v1", + "schema": "polylogue.raw-replay-application-receipt.v2", "source_rows": source, "membership_rows": memberships, "application_rows": applications, + "head_rows": heads, + "session_rows": sessions, } ) +def validate_raw_replay_application_receipt( + plan: RawReplayPlan, + receipt: Mapping[str, object], +) -> tuple[bool, tuple[str, ...]]: + """Prove exact replay postconditions; parsed timestamps are never sufficient.""" + problems: list[str] = [] + if receipt.get("schema") != "polylogue.raw-replay-application-receipt.v2": + problems.append("application receipt schema is not v2") + + def rows(name: str) -> list[Mapping[str, object]]: + value = receipt.get(name) + if not isinstance(value, list) or any(not isinstance(row, dict) for row in value): + problems.append(f"{name} is not a row list") + return [] + return value + + source_rows = rows("source_rows") + membership_rows = rows("membership_rows") + application_rows = rows("application_rows") + head_rows = rows("head_rows") + session_rows = rows("session_rows") + source_ids = {str(row.get("raw_id")) for row in source_rows} + if source_ids != set(plan.input_raw_ids): + problems.append("source receipt raw ids do not match the immutable plan") + if any(row.get("parsed_at_ms") is None or row.get("parse_error") is not None for row in source_rows): + problems.append("source receipt contains an unparsed or parse-failed raw") + expected_keys = set(plan.logical_keys) + head_keys = {str(row.get("logical_source_key")) for row in head_rows} + if not expected_keys: + problems.append("executed replay plan has no logical authority keys") + elif head_keys != expected_keys: + problems.append("accepted head keys do not match the immutable plan") + terminal_keys = { + str(row.get("logical_source_key")) + for row in application_rows + if row.get("decision") in {"selected_baseline", "applied_append", "superseded"} + } | { + str(row.get("logical_source_key")) + for row in membership_rows + if row.get("decision") in {"applied", "superseded_equivalent", "superseded_prefix"} + } + if not expected_keys.issubset(terminal_keys): + problems.append("not every logical key has a terminal application or membership decision") + head_session_ids = {str(row.get("session_id")) for row in head_rows} + session_ids = {str(row.get("session_id")) for row in session_rows} + if head_session_ids != session_ids: + problems.append("accepted head sessions do not match materialized session rows") + session_content = {str(row.get("session_id")): str(row.get("content_hash")) for row in session_rows} + if any( + str(row.get("accepted_content_hash")) != session_content.get(str(row.get("session_id"))) for row in head_rows + ): + problems.append("accepted head content hashes do not match materialized sessions") + input_raw_ids = set(plan.input_raw_ids) + if any(str(row.get("accepted_raw_id")) not in input_raw_ids for row in head_rows): + problems.append("accepted heads do not point into the immutable input component") + return not problems, tuple(problems) + + def record_raw_replay_outcome( archive_root: Path, census_id: str, @@ -622,8 +764,10 @@ def record_raw_replay_outcome( """ UPDATE raw_authority_census_plans SET outcome_status = ?, reason = ?, next_action = ?, - application_receipt_json = ?, recorded_at_ms = ? + application_receipt_json = ?, outcome_recorded = 1, + recorded_at_ms = ? WHERE census_id = ? AND plan_id = ? AND selected = 1 + AND outcome_recorded = 0 """, ( outcome.status.value, @@ -639,6 +783,189 @@ def record_raw_replay_outcome( raise RuntimeError(f"outcome does not conserve one selected plan: {outcome.plan_id}") +def _raw_replay_plan_from_row(row: sqlite3.Row) -> RawReplayPlan: + return RawReplayPlan( + plan_id=str(row["plan_id"]), + input_digest=str(row["input_digest"]), + input_raw_ids=tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))), + logical_keys=tuple(str(value) for value in json.loads(str(row["logical_keys_json"]))), + authority_witness=json_document(json.loads(str(row["authority_witness_json"]))), + source_preconditions=json_document(json.loads(str(row["source_preconditions_json"]))), + index_preconditions=json_document(json.loads(str(row["index_preconditions_json"]))), + ) + + +def _raw_authority_census_receipt(conn: sqlite3.Connection, census_id: str) -> RawAuthorityCensusReceipt: + row = conn.execute( + """ + SELECT census_id, sequence_no, inventory_digest, residual_digest, + plan_count, executable_plan_count, residual_plan_count, + predecessor_census_id, mode, lifecycle_status, quiescent, + fixed_point + FROM raw_authority_censuses WHERE census_id = ? + """, + (census_id,), + ).fetchone() + if row is None: + raise KeyError(census_id) + return RawAuthorityCensusReceipt( + census_id=str(row[0]), + sequence_no=int(row[1]), + inventory_digest=str(row[2]), + residual_digest=str(row[3]), + plan_count=int(row[4]), + executable_plan_count=int(row[5]), + residual_plan_count=int(row[6]), + predecessor_census_id=str(row[7]) if row[7] is not None else None, + mode=str(row[8]), + lifecycle_status=str(row[9]), + quiescent=bool(row[10]), + fixed_point=bool(row[11]), + ) + + +def finalize_raw_authority_census( + archive_root: Path, + census_id: str, + *, + interrupted: bool = False, +) -> RawAuthorityCensusReceipt: + """Publish a census only after every selected plan has a recorded outcome.""" + now = int(time.time() * 1000) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + status = conn.execute( + "SELECT lifecycle_status FROM raw_authority_censuses WHERE census_id = ?", + (census_id,), + ).fetchone() + if status is None: + raise KeyError(census_id) + if str(status[0]) != "planned": + return _raw_authority_census_receipt(conn, census_id) + pending = int( + conn.execute( + """ + SELECT COUNT(*) FROM raw_authority_census_plans + WHERE census_id = ? AND selected = 1 AND outcome_recorded = 0 + """, + (census_id,), + ).fetchone()[0] + ) + if pending: + raise RuntimeError(f"raw authority census still has {pending} pending selected outcome(s)") + conn.execute( + """ + UPDATE raw_authority_censuses + SET lifecycle_status = ?, completed_at_ms = ? + WHERE census_id = ? AND lifecycle_status = 'planned' + """, + ("interrupted" if interrupted else "completed", now, census_id), + ) + return _raw_authority_census_receipt(conn, census_id) + + +def recover_interrupted_raw_authority_censuses(archive_root: Path) -> int: + """Reconcile unfinished apply censuses from durable postconditions.""" + source_db = archive_root / "source.db" + if not source_db.is_file(): + return 0 + with closing(sqlite3.connect(source_db)) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + """ + SELECT c.census_id, p.* + FROM raw_authority_censuses AS c + JOIN raw_authority_census_plans AS cp ON cp.census_id = c.census_id + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE c.lifecycle_status = 'planned' + AND cp.selected = 1 AND cp.outcome_recorded = 0 + ORDER BY c.sequence_no, cp.ordinal + """ + ).fetchall() + census_ids = tuple( + str(row[0]) + for row in conn.execute( + "SELECT census_id FROM raw_authority_censuses WHERE lifecycle_status = 'planned' ORDER BY sequence_no" + ) + ) + for row in rows: + census_id = str(row["census_id"]) + plan = _raw_replay_plan_from_row(row) + receipt = raw_replay_application_receipt(archive_root, plan) + valid_receipt, problems = validate_raw_replay_application_receipt(plan, receipt) + if valid_receipt: + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.EXECUTED, + "interrupted application recovered from exact durable postconditions", + "none", + receipt, + ) + record_raw_replay_outcome(archive_root, census_id, outcome) + continue + valid_plan, observed = validate_raw_replay_plan(archive_root, plan) + if not valid_plan: + reject_stale_raw_replay_plan(archive_root, census_id, plan, observed) + else: + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.RETRYABLE, + "interrupted before exact application postconditions were durable: " + "; ".join(problems), + "retry the same immutable plan", + receipt, + ) + record_raw_replay_outcome(archive_root, census_id, outcome) + for census_id in census_ids: + finalize_raw_authority_census(archive_root, census_id, interrupted=True) + return len(census_ids) + + +def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolution: str) -> JSONDocument: + """Explicitly acknowledge current evidence and reopen replanning.""" + if not resolution.strip(): + raise ValueError("raw authority blocker resolution must be non-empty") + source_db = archive_root / "source.db" + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute( + """ + SELECT b.blocker_id, b.plan_id, b.expected_json, p.input_raw_ids_json + FROM raw_authority_blockers AS b + JOIN raw_authority_plans AS p ON p.plan_id = b.plan_id + WHERE b.blocker_id = ? AND b.resolved_at_ms IS NULL + """, + (blocker_id,), + ).fetchone() + if row is None: + raise KeyError(blocker_id) + input_raw_ids = tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))) + observed = build_raw_replay_plans(archive_root, (input_raw_ids,))[0] + now = int(time.time() * 1000) + receipt = json_document( + { + "schema": "polylogue.raw-authority-blocker-resolution.v1", + "blocker_id": blocker_id, + "superseded_plan_id": str(row["plan_id"]), + "current_plan": observed.to_dict(), + "operator_resolution": resolution.strip(), + "resolved_at_ms": now, + } + ) + with closing(sqlite3.connect(source_db)) as conn, conn: + updated = conn.execute( + """ + UPDATE raw_authority_blockers + SET resolved_at_ms = ?, resolution = ? + WHERE blocker_id = ? AND resolved_at_ms IS NULL + """, + (now, _canonical_json(receipt), blocker_id), + ).rowcount + if updated != 1: + raise RuntimeError(f"raw authority blocker changed during resolution: {blocker_id}") + return receipt + + def reject_stale_raw_replay_plan( archive_root: Path, census_id: str, @@ -679,8 +1006,10 @@ def reject_stale_raw_replay_plan( """ UPDATE raw_authority_census_plans SET outcome_status = 'rejected_stale', reason = ?, next_action = ?, - application_receipt_json = ?, recorded_at_ms = ? + application_receipt_json = ?, outcome_recorded = 1, + recorded_at_ms = ? WHERE census_id = ? AND plan_id = ? AND selected = 1 + AND outcome_recorded = 0 """, ( outcome.reason, @@ -696,6 +1025,67 @@ def reject_stale_raw_replay_plan( return outcome +def reject_invalid_raw_replay_application( + archive_root: Path, + census_id: str, + plan: RawReplayPlan, + receipt: JSONDocument, + problems: Sequence[str], +) -> RawReplayPlanOutcome: + """Fail closed when a writer returns without exact application postconditions.""" + now = int(time.time() * 1000) + observed = json_document({"application_receipt": receipt, "problems": list(problems)}) + blocker_id = f"raw-authority-blocker:{_digest([plan.plan_id, observed])}" + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.REJECTED_STALE, + "raw replay application did not satisfy exact durable postconditions", + "resolve the durable raw-authority blocker before automatic convergence resumes", + json_document({"expected": plan.to_dict(), "observed": observed, "blocker_id": blocker_id}), + ) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + conn.execute( + """ + INSERT INTO raw_authority_blockers ( + blocker_id, plan_id, census_id, reason, expected_json, + observed_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(blocker_id) DO NOTHING + """, + ( + blocker_id, + plan.plan_id, + census_id, + outcome.reason, + _canonical_json(plan.to_dict()), + _canonical_json(observed), + now, + ), + ) + updated = conn.execute( + """ + UPDATE raw_authority_census_plans + SET outcome_status = 'rejected_stale', reason = ?, next_action = ?, + application_receipt_json = ?, outcome_recorded = 1, + recorded_at_ms = ? + WHERE census_id = ? AND plan_id = ? AND selected = 1 + AND outcome_recorded = 0 + """, + ( + outcome.reason, + outcome.next_action, + _canonical_json(outcome.application_receipt or {}), + now, + census_id, + plan.plan_id, + ), + ).rowcount + if updated != 1: + raise RuntimeError(f"invalid application does not conserve one selected plan: {plan.plan_id}") + return outcome + + __all__ = [ "RAW_AUTHORITY_CENSUS_QUERY_PREFIX", "RAW_AUTHORITY_PARSER_FINGERPRINT", @@ -705,13 +1095,18 @@ def reject_stale_raw_replay_plan( "RawReplayPlanStatus", "build_raw_replay_plan", "build_raw_replay_plans", + "finalize_raw_authority_census", "raw_replay_application_receipt", "raw_authority_census_query_handle", "raw_replay_plan_last_attempts", + "recover_interrupted_raw_authority_censuses", "read_raw_authority_census", "record_raw_authority_census", "record_raw_replay_outcome", + "reject_invalid_raw_replay_application", "reject_stale_raw_replay_plan", + "resolve_raw_authority_blocker", "unresolved_raw_authority_blockers", "validate_raw_replay_plan", + "validate_raw_replay_application_receipt", ] diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 26fb2b4dcd..7250cc215e 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -62,12 +62,16 @@ RawReplayPlanOutcome, RawReplayPlanStatus, build_raw_replay_plans, + finalize_raw_authority_census, raw_replay_application_receipt, raw_replay_plan_last_attempts, record_raw_authority_census, record_raw_replay_outcome, + recover_interrupted_raw_authority_censuses, + reject_invalid_raw_replay_application, reject_stale_raw_replay_plan, unresolved_raw_authority_blockers, + validate_raw_replay_application_receipt, validate_raw_replay_plan, ) @@ -76,6 +80,7 @@ _PROBE_ONLY_EXACT_MESSAGE_ROW_LIMIT = 100_000 RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES = 1024 * 1024 * 1024 RAW_MATERIALIZATION_RESOURCE_BLOCK_REASON = "non-stream-safe raw payload exceeds the bounded replay limit" +RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT = 8 _TRANSIENT_LOCK_PARSE_ERROR = "OperationalError: database is locked" _QUARANTINED_ACCEPTED_RAW_REPAIR_DETAIL = "repair:accepted_quarantined_raw_exact_byte_and_semantic_proof" _QUARANTINED_ACCEPTED_RAW_REPAIR_LIMIT = 100 @@ -4591,6 +4596,12 @@ class RawMaterializationCandidates: expanded_raw_ids: tuple[str, ...] = () expanded_blob_bytes: dict[str, int] = field(default_factory=dict) authority_components: tuple[tuple[str, ...], ...] = () + missing_blob_raw_ids: tuple[str, ...] = () + adoption_deferred_raw_ids: tuple[str, ...] = () + authority_quarantined_raw_ids: tuple[str, ...] = () + byte_authority_fragment_raw_ids: tuple[str, ...] = () + byte_authority_quarantined_raw_ids: tuple[str, ...] = () + byte_authority_pending_raw_ids: tuple[str, ...] = () @property def total_blob_bytes(self) -> int: @@ -4657,6 +4668,12 @@ def _raw_materialization_candidate_ids( missing_blob_source_available = 0 missing_blob_source_missing = 0 already_parsed = 0 + missing_blob_raw_ids: list[str] = [] + adoption_deferred_raw_ids: list[str] = [] + authority_quarantined_raw_ids: list[str] = [] + byte_authority_fragment_raw_ids: list[str] = [] + byte_authority_quarantined_raw_ids: list[str] = [] + byte_authority_pending_raw_ids: list[str] = [] expanded_raw_ids: tuple[str, ...] = () expanded_blob_bytes: dict[str, int] = {} authority_components: tuple[tuple[str, ...], ...] = () @@ -4781,8 +4798,10 @@ def _raw_materialization_candidate_ids( byte_authority_quarantined = 0 byte_authority_pending = 0 for row in rows: + row_raw_id = str(row["raw_id"]) if bool(row["adoption_deferred"]): adoption_deferred += 1 + adoption_deferred_raw_ids.append(row_raw_id) continue if bool(row["application_terminal"]): continue @@ -4790,15 +4809,19 @@ def _raw_materialization_candidate_ids( continue if bool(row["membership_authority_quarantined"]): authority_quarantined += 1 + authority_quarantined_raw_ids.append(row_raw_id) continue if bool(row["byte_authority_fragment"]): byte_authority_fragments += 1 + byte_authority_fragment_raw_ids.append(row_raw_id) continue if bool(row["byte_authority_quarantined"]): byte_authority_quarantined += 1 + byte_authority_quarantined_raw_ids.append(row_raw_id) continue if bool(row["byte_authority_pending"]): byte_authority_pending += 1 + byte_authority_pending_raw_ids.append(row_raw_id) continue if row["parse_error"] and not _raw_materialization_retryable_missing_blob_error(row["parse_error"]): continue @@ -4820,6 +4843,7 @@ def _raw_materialization_candidate_ids( already_parsed += 1 else: missing_blobs += 1 + missing_blob_raw_ids.append(row_raw_id) if _raw_materialization_source_available(str(row["source_path"] or "")): missing_blob_source_available += 1 else: @@ -4855,6 +4879,12 @@ def _raw_materialization_candidate_ids( expanded_raw_ids=expanded_raw_ids, expanded_blob_bytes=expanded_blob_bytes, authority_components=authority_components, + missing_blob_raw_ids=tuple(sorted(missing_blob_raw_ids)), + adoption_deferred_raw_ids=tuple(sorted(adoption_deferred_raw_ids)), + authority_quarantined_raw_ids=tuple(sorted(authority_quarantined_raw_ids)), + byte_authority_fragment_raw_ids=tuple(sorted(byte_authority_fragment_raw_ids)), + byte_authority_quarantined_raw_ids=tuple(sorted(byte_authority_quarantined_raw_ids)), + byte_authority_pending_raw_ids=tuple(sorted(byte_authority_pending_raw_ids)), ) @@ -4920,6 +4950,66 @@ def _raw_materialization_component_blob_bytes(candidates: RawMaterializationCand return candidates.expanded_blob_bytes.get(raw_id, candidates.raw_blob_bytes.get(raw_id, 0)) +def _raw_authority_scope( + *, + raw_artifact_id: str | None, + provider: str | None, + source_family: str | None, + source_root: Path | None, + raw_artifact_limit: int | None, +) -> dict[str, object]: + return { + "raw_artifact_id": raw_artifact_id, + "provider": provider, + "source_family": source_family, + "source_root": str(source_root) if source_root is not None else None, + "raw_artifact_limit": raw_artifact_limit, + } + + +def _raw_authority_residual( + candidates: RawMaterializationCandidates, + *, + census_pending_raw_ids: tuple[str, ...] = (), + resource_blocked_plan_ids: tuple[str, ...] = (), +) -> dict[str, object]: + """Return identity-sensitive residual debt for fixed-point comparison.""" + return { + "missing_blob_raw_ids": list(candidates.missing_blob_raw_ids), + "adoption_deferred_raw_ids": list(candidates.adoption_deferred_raw_ids), + "authority_quarantined_raw_ids": list(candidates.authority_quarantined_raw_ids), + "byte_authority_fragment_raw_ids": list(candidates.byte_authority_fragment_raw_ids), + "byte_authority_quarantined_raw_ids": list(candidates.byte_authority_quarantined_raw_ids), + "byte_authority_pending_raw_ids": list(candidates.byte_authority_pending_raw_ids), + "census_pending_raw_ids": list(census_pending_raw_ids), + "resource_blocked_plan_ids": list(resource_blocked_plan_ids), + } + + +def _raw_materialization_base_metrics( + candidates: RawMaterializationCandidates, + *, + recovered_census_count: int, +) -> dict[str, float]: + metrics = { + "raw_materialization_candidate_count": float(len(candidates.raw_ids)), + "raw_materialization_missing_blob_count": float(candidates.missing_blobs), + "raw_materialization_missing_blob_source_available_count": float(candidates.missing_blob_source_available), + "raw_materialization_missing_blob_source_missing_count": float(candidates.missing_blob_source_missing), + "raw_materialization_already_parsed_count": float(candidates.already_parsed), + "raw_materialization_total_blob_bytes": float(candidates.total_blob_bytes), + "raw_materialization_max_blob_bytes": float(candidates.max_blob_bytes), + "raw_materialization_adoption_deferred_count": float(candidates.adoption_deferred), + "raw_materialization_authority_quarantined_count": float(candidates.authority_quarantined), + "raw_materialization_byte_authority_fragment_count": float(candidates.byte_authority_fragments), + "raw_materialization_byte_authority_quarantined_count": float(candidates.byte_authority_quarantined), + "raw_materialization_byte_authority_pending_count": float(candidates.byte_authority_pending), + } + if recovered_census_count: + metrics["raw_materialization_recovered_census_count"] = float(recovered_census_count) + return metrics + + def _raw_replay_plan_outcome( conn: sqlite3.Connection, plan: RawReplayPlan, @@ -5520,7 +5610,11 @@ def to_dict(self) -> JSONDocument: "metrics": dict(self.metrics), } if self.plan_outcomes: - payload["plan_outcomes"] = [outcome.to_dict() for outcome in self.plan_outcomes] + payload["plan_outcome_count"] = len(self.plan_outcomes) + payload["plan_outcomes"] = [ + outcome.to_dict() for outcome in self.plan_outcomes[:RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT] + ] + payload["plan_outcomes_truncated"] = len(self.plan_outcomes) > RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT if self.census_receipt is not None: payload["census"] = { "census_id": self.census_receipt.census_id, @@ -5531,6 +5625,9 @@ def to_dict(self) -> JSONDocument: "executable_plan_count": self.census_receipt.executable_plan_count, "residual_plan_count": self.census_receipt.residual_plan_count, "predecessor_census_id": self.census_receipt.predecessor_census_id, + "mode": self.census_receipt.mode, + "lifecycle_status": self.census_receipt.lifecycle_status, + "quiescent": self.census_receipt.quiescent, "fixed_point": self.census_receipt.fixed_point, "query_handle": self.census_receipt.query_handle, } @@ -6350,6 +6447,7 @@ def repair_raw_materialization( ) -> RepairResult: """Converge retained raws through typed per-session revision authority.""" archive_root = _raw_materialization_archive_root(config) + recovered_census_count = recover_interrupted_raw_authority_censuses(archive_root) blocker_count = unresolved_raw_authority_blockers(archive_root) if blocker_count: return _internal_derived_repair_result( @@ -6368,15 +6466,21 @@ def repair_raw_materialization( source_family=source_family, source_root=source_root, ) - census_failed_raw_ids: set[str] = set() - if candidates.raw_ids: - from polylogue.sources.revision_backfill import census_historical_revision_evidence + from polylogue.sources.revision_backfill import ( + RawRevisionReplayResourceBlockedError, + census_historical_revision_evidence, + uncensused_historical_revision_raw_ids, + ) + relevant_raw_ids = list(candidates.expanded_raw_ids or tuple(candidates.raw_ids)) + uncensused_raw_ids = set(uncensused_historical_revision_raw_ids(archive_root, relevant_raw_ids)) + census_failed_raw_ids: set[str] = set() + census_resource_blocked_raw_ids: set[str] = set() + if uncensused_raw_ids: preliminary_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) - preliminary_selected = ( - preliminary_components[:raw_artifact_limit] if raw_artifact_limit is not None else preliminary_components - ) - for component in preliminary_selected: + for component in preliminary_components: + if not uncensused_raw_ids.intersection(component): + continue seed = _raw_materialization_component_seed(candidates, component) try: census_historical_revision_evidence( @@ -6384,9 +6488,14 @@ def repair_raw_materialization( selected_raw_ids=[seed], max_payload_bytes=RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES, ) + except RawRevisionReplayResourceBlockedError as exc: + logger.warning( + "raw authority census resource-blocked for component containing %s: %s", + seed, + exc, + ) + census_resource_blocked_raw_ids.update(component) except Exception: - # The immutable plan below conserves the component as retryable; - # a census failure must not make it disappear from inventory. logger.exception("raw authority census failed for component containing %s", seed) census_failed_raw_ids.update(component) candidates = _raw_materialization_candidate_ids( @@ -6396,6 +6505,77 @@ def repair_raw_materialization( source_family=source_family, source_root=source_root, ) + relevant_raw_ids = list(candidates.expanded_raw_ids or tuple(candidates.raw_ids)) + uncensused_raw_ids = set(uncensused_historical_revision_raw_ids(archive_root, relevant_raw_ids)) + census_pending_raw_ids = tuple(sorted(uncensused_raw_ids | census_failed_raw_ids)) + if census_pending_raw_ids: + residual = _raw_authority_residual(candidates, census_pending_raw_ids=census_pending_raw_ids) + census_receipt = record_raw_authority_census( + archive_root, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="census", + quiescent=False, + scope=_raw_authority_scope( + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + raw_artifact_limit=raw_artifact_limit, + ), + residual=residual, + ) + metrics = _raw_materialization_base_metrics( + candidates, + recovered_census_count=recovered_census_count, + ) + metrics.update( + { + "raw_materialization_selected_count": 0.0, + "raw_materialization_selected_total_blob_bytes": 0.0, + "raw_materialization_selected_max_blob_bytes": 0.0, + "raw_materialization_executed_count": 0.0, + "raw_materialization_census_incomplete_raw_count": float(len(census_pending_raw_ids)), + "raw_materialization_census_sequence": float(census_receipt.sequence_no), + "raw_materialization_census_fixed_point": 0.0, + } + ) + if census_resource_blocked_raw_ids: + metrics["raw_materialization_resource_blocked_count"] = float(len(census_resource_blocked_raw_ids)) + metrics["raw_materialization_execute_blob_limit_bytes"] = float( + RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + ) + oversized = { + raw_id + for raw_id in census_resource_blocked_raw_ids + if _raw_materialization_component_blob_bytes(candidates, raw_id) + > RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + } + if oversized: + metrics["raw_materialization_oversized_count"] = float(len(oversized)) + stream_oversized = { + raw_id for raw_id in oversized if _raw_materialization_stream_safe(candidates, raw_id) + } + if stream_oversized: + metrics["raw_materialization_stream_oversized_count"] = float(len(stream_oversized)) + detail = ( + f"Raw replay planning paused until the persisted parser census completes for " + f"{len(census_pending_raw_ids):,} relevant raw(s)" + ) + if census_resource_blocked_raw_ids: + detail += ( + f"; {len(census_resource_blocked_raw_ids):,} raw(s) belong to authority components whose " + f"aggregate payload exceeds {_format_bytes(RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES)}" + ) + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=detail, + metrics=metrics, + census_receipt=census_receipt, + ) candidate_raw_ids = candidates.raw_ids ordered_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) plans = build_raw_replay_plans(archive_root, ordered_components) @@ -6411,9 +6591,7 @@ def repair_raw_materialization( ordered_components[:raw_artifact_limit] if raw_artifact_limit is not None else ordered_components ) blocked_components = [ - component - for component in selected_components - if all_blocked_component_raw_ids.intersection(component) or census_failed_raw_ids.intersection(component) + component for component in selected_components if all_blocked_component_raw_ids.intersection(component) ] blocked_component_raw_ids = {raw_id for component in blocked_components for raw_id in component} blocked_plan_outcomes = tuple( @@ -6421,16 +6599,8 @@ def repair_raw_materialization( plan_by_component[component].plan_id, component, RawReplayPlanStatus.RETRYABLE, - ( - "authority component census did not complete" - if census_failed_raw_ids.intersection(component) - else "authority component exceeds the bounded replay resource envelope" - ), - ( - "resume the bounded source census before replay planning" - if census_failed_raw_ids.intersection(component) - else "retry the same plan after streaming/resource admission is available" - ), + "authority component exceeds the bounded replay resource envelope", + "retry the same plan after streaming/resource admission is available", ) for component in blocked_components ) @@ -6449,31 +6619,20 @@ def repair_raw_materialization( (_raw_materialization_component_blob_bytes(candidates, raw_id) for raw_id in selected_component_raw_ids), default=0, ) - metrics = { - "raw_materialization_candidate_count": float(len(candidate_raw_ids)), - "raw_materialization_selected_count": float(len(selected_candidate_raw_ids)), - "raw_materialization_missing_blob_count": float(missing_blobs), - "raw_materialization_missing_blob_source_available_count": float(candidates.missing_blob_source_available), - "raw_materialization_missing_blob_source_missing_count": float(candidates.missing_blob_source_missing), - "raw_materialization_already_parsed_count": float(candidates.already_parsed), - "raw_materialization_total_blob_bytes": float(candidates.total_blob_bytes), - "raw_materialization_max_blob_bytes": float(candidates.max_blob_bytes), - "raw_materialization_selected_total_blob_bytes": float(selected_total_bytes), - "raw_materialization_selected_max_blob_bytes": float(selected_max_bytes), - "raw_materialization_adoption_deferred_count": float(candidates.adoption_deferred), - "raw_materialization_authority_quarantined_count": float(candidates.authority_quarantined), - "raw_materialization_byte_authority_fragment_count": float(candidates.byte_authority_fragments), - "raw_materialization_byte_authority_quarantined_count": float(candidates.byte_authority_quarantined), - "raw_materialization_byte_authority_pending_count": float(candidates.byte_authority_pending), - } + metrics = _raw_materialization_base_metrics(candidates, recovered_census_count=recovered_census_count) + metrics.update( + { + "raw_materialization_selected_count": float(len(selected_candidate_raw_ids)), + "raw_materialization_selected_total_blob_bytes": float(selected_total_bytes), + "raw_materialization_selected_max_blob_bytes": float(selected_max_bytes), + } + ) if raw_artifact_limit is not None: metrics["raw_materialization_limit"] = float(raw_artifact_limit) metrics["raw_materialization_selected_component_count"] = float(len(selected_components)) metrics["raw_materialization_before_component_count"] = float(len(ordered_components)) metrics["raw_materialization_selected_executable_component_count"] = float(len(executable_components)) metrics["raw_materialization_selected_blocked_component_count"] = float(len(blocked_components)) - if census_failed_raw_ids: - metrics["raw_materialization_census_incomplete_raw_count"] = float(len(census_failed_raw_ids)) if all_blocked_component_raw_ids: metrics["raw_materialization_resource_blocked_count"] = float(len(all_blocked_component_raw_ids)) metrics["raw_materialization_execute_blob_limit_bytes"] = float(RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES) @@ -6499,28 +6658,32 @@ def repair_raw_materialization( if oversized_stream_safe_raw_ids: metrics["raw_materialization_stream_oversized_count"] = float(len(oversized_stream_safe_raw_ids)) selected_plan_ids = {plan_by_component[component].plan_id for component in selected_components} + executable_plan_ids = { + plan_by_component[component].plan_id + for component in ordered_components + if not all_blocked_component_raw_ids.intersection(component) + } + residual = _raw_authority_residual( + candidates, + resource_blocked_plan_ids=tuple( + sorted(plan_by_component[component].plan_id for component in all_blocked_components) + ), + ) census_receipt = record_raw_authority_census( archive_root, plans, selected_plan_ids=set() if dry_run else selected_plan_ids, - executable_plan_ids={plan.plan_id for plan in plans}, - scope={ - "raw_artifact_id": raw_artifact_id, - "provider": provider, - "source_family": source_family, - "source_root": str(source_root) if source_root is not None else None, - "raw_artifact_limit": raw_artifact_limit, - }, - residual={ - "missing_blobs": missing_blobs, - "missing_blob_source_available": candidates.missing_blob_source_available, - "missing_blob_source_missing": candidates.missing_blob_source_missing, - "adoption_deferred": candidates.adoption_deferred, - "authority_quarantined": candidates.authority_quarantined, - "byte_authority_fragments": candidates.byte_authority_fragments, - "byte_authority_quarantined": candidates.byte_authority_quarantined, - "byte_authority_pending": candidates.byte_authority_pending, - }, + executable_plan_ids=executable_plan_ids, + mode="dry_run" if dry_run else "apply", + quiescent=True, + scope=_raw_authority_scope( + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + raw_artifact_limit=raw_artifact_limit, + ), + residual=residual, ) metrics["raw_materialization_census_sequence"] = float(census_receipt.sequence_no) metrics["raw_materialization_census_fixed_point"] = float(census_receipt.fixed_point) @@ -6630,6 +6793,7 @@ def repair_raw_materialization( ] for outcome in carried: record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + census_receipt = finalize_raw_authority_census(archive_root, census_receipt.census_id) plan_outcomes = tuple(stale_outcomes + carried) + blocked_plan_outcomes metrics["raw_materialization_plan_rejected_stale_count"] = float(len(stale_outcomes)) metrics["raw_materialization_plan_carried_forward_count"] = float(len(carried)) @@ -6648,11 +6812,7 @@ def repair_raw_materialization( census_receipt=census_receipt, ) - from polylogue.sources.revision_backfill import ( - RawRevisionReplayResourceBlockedError, - RevisionBackfillResult, - backfill_historical_revision_evidence, - ) + from polylogue.sources.revision_backfill import RevisionBackfillResult, backfill_historical_revision_evidence executable_raw_ids = raw_ids metrics["raw_materialization_executed_count"] = float(len(executable_raw_ids)) @@ -6702,11 +6862,30 @@ def repair_raw_materialization( ) component_outcomes = _raw_replay_plan_outcomes(archive_root, [plan], remaining=current) for outcome in component_outcomes: - receipted = dataclasses.replace( - outcome, - application_receipt=raw_replay_application_receipt(archive_root, plan), - ) - record_raw_replay_outcome(archive_root, census_receipt.census_id, receipted) + application_receipt = raw_replay_application_receipt(archive_root, plan) + receipted = dataclasses.replace(outcome, application_receipt=application_receipt) + if outcome.status is RawReplayPlanStatus.EXECUTED: + receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) + if not receipt_valid: + receipted = reject_invalid_raw_replay_application( + archive_root, + census_receipt.census_id, + plan, + application_receipt, + receipt_problems, + ) + else: + record_raw_replay_outcome(archive_root, census_receipt.census_id, receipted) + elif outcome.status is RawReplayPlanStatus.REJECTED_STALE: + receipted = reject_invalid_raw_replay_application( + archive_root, + census_receipt.census_id, + plan, + application_receipt, + (outcome.reason,), + ) + else: + record_raw_replay_outcome(archive_root, census_receipt.census_id, receipted) execution_outcomes.append(receipted) replay = RevisionBackfillResult( @@ -6740,6 +6919,7 @@ def repair_raw_materialization( } ) plan_outcomes = tuple(execution_outcomes) + blocked_plan_outcomes + census_receipt = finalize_raw_authority_census(archive_root, census_receipt.census_id) for status in RawReplayPlanStatus: metrics[f"raw_materialization_plan_{status.value}_count"] = float( sum(outcome.status is status for outcome in plan_outcomes) diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index b554a4a71b..d1728660fd 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -104,11 +104,24 @@ -- Durable authority reconciliation ledger. The source tier owns this -- evidence because index.db and ops.db are rebuildable/disposable: neither -- can be the authority for whether an accepted replay plan was conserved. +CREATE TABLE IF NOT EXISTS raw_authority_parser_census ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + parser_fingerprint TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('complete', 'failed')), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + detail TEXT NOT NULL DEFAULT '', + censused_at_ms INTEGER NOT NULL CHECK(censused_at_ms >= 0) +) STRICT; + CREATE TABLE IF NOT EXISTS raw_authority_censuses ( census_id TEXT PRIMARY KEY, sequence_no INTEGER NOT NULL UNIQUE CHECK(sequence_no > 0), scope_json TEXT NOT NULL CHECK(json_valid(scope_json)), + residual_json TEXT NOT NULL CHECK(json_valid(residual_json)), parser_fingerprint TEXT NOT NULL, + mode TEXT NOT NULL CHECK(mode IN ('census', 'dry_run', 'apply')), + lifecycle_status TEXT NOT NULL CHECK(lifecycle_status IN ('planned', 'completed', 'interrupted')), + quiescent INTEGER NOT NULL CHECK(quiescent IN (0, 1)), inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), plan_count INTEGER NOT NULL CHECK(plan_count >= 0), @@ -117,9 +130,13 @@ predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), fixed_point INTEGER NOT NULL DEFAULT 0 CHECK(fixed_point IN (0, 1)), created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), - completed_at_ms INTEGER NOT NULL CHECK(completed_at_ms >= created_at_ms), + completed_at_ms INTEGER CHECK(completed_at_ms IS NULL OR completed_at_ms >= created_at_ms), CHECK(plan_count >= executable_plan_count), - CHECK(plan_count >= residual_plan_count) + CHECK(plan_count >= residual_plan_count), + CHECK( + (lifecycle_status = 'planned' AND completed_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) + ) ) STRICT; CREATE TABLE IF NOT EXISTS raw_authority_plans ( @@ -145,6 +162,7 @@ reason TEXT NOT NULL, next_action TEXT NOT NULL, application_receipt_json TEXT NOT NULL DEFAULT '{{}}' CHECK(json_valid(application_receipt_json)), + outcome_recorded INTEGER NOT NULL DEFAULT 0 CHECK(outcome_recorded IN (0, 1)), recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0), PRIMARY KEY(census_id, plan_id), UNIQUE(census_id, ordinal) diff --git a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql index 012fa0ac78..908eb76eb3 100644 --- a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql +++ b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql @@ -1,10 +1,23 @@ -- migration-safety: additive-no-backup -- Durable, restart-safe conservation ledger for raw authority reconciliation. +CREATE TABLE raw_authority_parser_census ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + parser_fingerprint TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('complete', 'failed')), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + detail TEXT NOT NULL DEFAULT '', + censused_at_ms INTEGER NOT NULL CHECK(censused_at_ms >= 0) +) STRICT; + CREATE TABLE raw_authority_censuses ( census_id TEXT PRIMARY KEY, sequence_no INTEGER NOT NULL UNIQUE CHECK(sequence_no > 0), scope_json TEXT NOT NULL CHECK(json_valid(scope_json)), + residual_json TEXT NOT NULL CHECK(json_valid(residual_json)), parser_fingerprint TEXT NOT NULL, + mode TEXT NOT NULL CHECK(mode IN ('census', 'dry_run', 'apply')), + lifecycle_status TEXT NOT NULL CHECK(lifecycle_status IN ('planned', 'completed', 'interrupted')), + quiescent INTEGER NOT NULL CHECK(quiescent IN (0, 1)), inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), plan_count INTEGER NOT NULL CHECK(plan_count >= 0), @@ -13,9 +26,13 @@ CREATE TABLE raw_authority_censuses ( predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), fixed_point INTEGER NOT NULL DEFAULT 0 CHECK(fixed_point IN (0, 1)), created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), - completed_at_ms INTEGER NOT NULL CHECK(completed_at_ms >= created_at_ms), + completed_at_ms INTEGER CHECK(completed_at_ms IS NULL OR completed_at_ms >= created_at_ms), CHECK(plan_count >= executable_plan_count), - CHECK(plan_count >= residual_plan_count) + CHECK(plan_count >= residual_plan_count), + CHECK( + (lifecycle_status = 'planned' AND completed_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) + ) ) STRICT; CREATE TABLE raw_authority_plans ( @@ -41,6 +58,7 @@ CREATE TABLE raw_authority_census_plans ( reason TEXT NOT NULL, next_action TEXT NOT NULL, application_receipt_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(application_receipt_json)), + outcome_recorded INTEGER NOT NULL DEFAULT 0 CHECK(outcome_recorded IN (0, 1)), recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0), PRIMARY KEY(census_id, plan_id), UNIQUE(census_id, ordinal) diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index 286e82e7b7..9f98778dea 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -43,6 +43,8 @@ def test_raw_authority_census_cli_resolves_receipt_handle( (), selected_plan_ids=set(), executable_plan_ids=set(), + mode="dry_run", + quiescent=True, scope={"source_family": "codex"}, residual={}, ) @@ -67,6 +69,37 @@ def test_raw_authority_census_cli_resolves_receipt_handle( assert payload["census"]["census_id"] == receipt.census_id +def test_raw_authority_blocker_resolution_cli_requires_confirmation( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + def resolve(_root: Path, blocker_id: str, *, resolution: str) -> dict[str, object]: + calls.append((blocker_id, resolution)) + return {"blocker_id": blocker_id, "current_plan": {"plan_id": "current-plan"}} + + monkeypatch.setattr("polylogue.storage.raw_authority.resolve_raw_authority_blocker", resolve) + base = [ + "--plain", + "ops", + "maintenance", + "raw-authority-blocker-resolve", + "--blocker-id", + "blocker-1", + "--reason", + "reviewed current evidence", + ] + refused = cli_runner.invoke(cli, base) + accepted = cli_runner.invoke(cli, [*base, "--yes"], catch_exceptions=False) + + assert refused.exit_code != 0 + assert "without --yes" in refused.output + assert accepted.exit_code == 0 + assert "Resolved blocker-1" in accepted.output + assert calls == [("blocker-1", "reviewed current evidence")] + + def _stage_uninitialized_archive(cli_workspace: dict[str, Path]) -> None: """Reset the workspace to an uninitialized state for plan/init tests. diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 4e9ccba7cf..92d6dd177f 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -691,6 +691,9 @@ def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: py executable_plan_count=1, residual_plan_count=2, predecessor_census_id="census:1:inventory:residual", + mode="apply", + lifecycle_status="completed", + quiescent=True, fixed_point=False, query_handle="polylogue://raw-authority-census/census:2:inventory:residual/0", ) @@ -715,6 +718,9 @@ def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: py "executable_plan_count": 1, "residual_plan_count": 2, "predecessor_census_id": census.predecessor_census_id, + "mode": "apply", + "lifecycle_status": "completed", + "quiescent": True, "fixed_point": False, "query_handle": census.query_handle, } diff --git a/tests/unit/mcp/test_server_surfaces.py b/tests/unit/mcp/test_server_surfaces.py index 4af4be7897..78c965fb99 100644 --- a/tests/unit/mcp/test_server_surfaces.py +++ b/tests/unit/mcp/test_server_surfaces.py @@ -570,6 +570,8 @@ def test_raw_authority_census_query_handle_resolves_bounded_ledger( (), selected_plan_ids=set(), executable_plan_ids=set(), + mode="dry_run", + quiescent=True, scope={"origin": "codex-session"}, residual={}, ) diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index e125d16fc9..f67ded3745 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -553,6 +553,7 @@ def test_source_publication_backfill_requires_verified_backup( ) -> None: db_path = workspace_env["archive_root"] / "source.db" with sqlite3.connect(db_path) as conn: + conn.execute("DROP TABLE raw_authority_parser_census") conn.execute("DROP TABLE raw_authority_blockers") conn.execute("DROP TABLE raw_authority_census_plans") conn.execute("DROP TABLE raw_authority_plans") @@ -584,6 +585,7 @@ def test_source_publication_backfill_requires_verified_backup( "sinex_publication_receipts", "excised_content", "raw_authority_censuses", + "raw_authority_parser_census", "raw_authority_plans", "raw_authority_census_plans", "raw_authority_blockers", diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 155a4e2d7b..d141cef5e8 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -1,23 +1,35 @@ from __future__ import annotations +import json import sqlite3 from pathlib import Path from typing import cast +from unittest.mock import patch import pytest from polylogue.config import Config from polylogue.core.enums import Provider +from polylogue.core.json import JSONDocument, json_document +from polylogue.maintenance.models import MaintenanceCategory from polylogue.sources.revision_backfill import census_historical_revision_evidence +from polylogue.storage import raw_authority as raw_authority_mod +from polylogue.storage import repair as repair_mod from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot from polylogue.storage.raw_authority import ( + RawReplayPlan, + RawReplayPlanOutcome, + RawReplayPlanStatus, build_raw_replay_plans, + finalize_raw_authority_census, read_raw_authority_census, record_raw_authority_census, + record_raw_replay_outcome, reject_stale_raw_replay_plan, + resolve_raw_authority_blocker, validate_raw_replay_plan, ) -from polylogue.storage.repair import repair_raw_materialization +from polylogue.storage.repair import RepairResult, repair_raw_materialization from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -88,7 +100,8 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat assert result.census_receipt is not None assert result.census_receipt.plan_count == 2 assert result.census_receipt.executable_plan_count == 2 - assert result.census_receipt.residual_plan_count == 1 + assert result.census_receipt.residual_plan_count == 0 + assert result.census_receipt.lifecycle_status == "completed" assert result.metrics["raw_materialization_plan_outcome_count"] == 2.0 assert result.metrics["raw_materialization_plan_carried_forward_count"] == 1.0 with sqlite3.connect(tmp_path / "source.db") as conn: @@ -117,6 +130,10 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat assert executed[0] == 1 assert '"application_rows"' in executed[2] assert '"membership_rows"' in executed[2] + application_receipt = json.loads(executed[2]) + assert application_receipt["application_rows"] + assert application_receipt["head_rows"] + assert application_receipt["session_rows"] assert plan_row is not None assert all(value not in (None, "", "[]", "{}") for value in plan_row) readiness = raw_materialization_readiness_snapshot(tmp_path) @@ -126,7 +143,8 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat assert census_status["residual_digest"] == result.census_receipt.residual_digest assert census_status["plan_count"] == 2 assert census_status["executable_plan_count"] == 2 - assert census_status["residual_plan_count"] == 1 + assert census_status["residual_plan_count"] == 0 + assert census_status["lifecycle_status"] == "completed" assert census_status["query_handle"] == result.census_receipt.query_handle first_page = read_raw_authority_census(tmp_path, result.census_receipt.query_handle, limit=1) assert first_page["returned_count"] == 1 @@ -165,6 +183,8 @@ def test_stale_plan_persists_blocker_before_automatic_replay_refuses_work(tmp_pa tmp_path, (plan,), selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, scope={"test": "stale"}, residual={}, ) @@ -222,6 +242,8 @@ def test_interrupted_census_has_no_partial_plan_visibility_and_retries_once(tmp_ tmp_path, plans, selected_plan_ids={plan.plan_id for plan in plans}, + mode="apply", + quiescent=True, scope={"test": "interruption"}, residual={}, ) @@ -236,6 +258,8 @@ def test_interrupted_census_has_no_partial_plan_visibility_and_retries_once(tmp_ tmp_path, plans, selected_plan_ids={plan.plan_id for plan in plans}, + mode="apply", + quiescent=True, scope={"test": "interruption"}, residual={}, ) @@ -248,3 +272,234 @@ def test_interrupted_census_has_no_partial_plan_visibility_and_retries_once(tmp_ ).fetchone()[0] == 2 ) + for plan in plans: + record_raw_replay_outcome( + tmp_path, + receipt.census_id, + RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.RETRYABLE, + "test interruption recovered", + "retry", + ), + ) + finalized = finalize_raw_authority_census(tmp_path, receipt.census_id, interrupted=True) + assert finalized.lifecycle_status == "interrupted" + + +def test_global_census_quiesces_moved_component_before_any_plan_is_published(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + first = _write_codex_raw( + tmp_path, + native_id="merged", + source_path="merged-old.jsonl", + acquired_at_ms=1, + text="old", + ) + second = _write_codex_raw( + tmp_path, + native_id="merged", + source_path="merged-new.jsonl", + acquired_at_ms=2, + text="new", + ) + third = _write_codex_raw( + tmp_path, + native_id="independent", + source_path="independent.jsonl", + acquired_at_ms=3, + ) + + preview = repair_raw_materialization(_config(tmp_path), dry_run=True, raw_artifact_limit=1) + + assert preview.census_receipt is not None + assert preview.census_receipt.quiescent is True + ledger = read_raw_authority_census(tmp_path, preview.census_receipt.query_handle) + raw_sets = { + frozenset(cast(list[str], cast(dict[str, object], cast(dict[str, object], item)["plan"])["input_raw_ids"])) + for item in cast(list[object], ledger["plans"]) + } + assert raw_sets == {frozenset((first, second)), frozenset((third,))} + + +def test_interrupted_apply_recovers_exact_durable_postconditions(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="crash", source_path="crash.jsonl", acquired_at_ms=1) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=RuntimeError("synthetic crash")): + with pytest.raises(RuntimeError, match="synthetic crash"): + repair_raw_materialization(_config(tmp_path)) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_censuses WHERE lifecycle_status = 'planned'").fetchone()[0] + == 1 + ) + recovered = repair_raw_materialization(_config(tmp_path)) + assert recovered.metrics["raw_materialization_recovered_census_count"] == 1.0 + with sqlite3.connect(tmp_path / "source.db") as conn: + row = conn.execute( + """ + SELECT c.lifecycle_status, cp.outcome_status + FROM raw_authority_censuses AS c + JOIN raw_authority_census_plans AS cp ON cp.census_id = c.census_id + WHERE c.lifecycle_status = 'interrupted' + """ + ).fetchone() + assert row == ("interrupted", "executed") + + +def test_parsed_timestamp_without_exact_application_receipt_fails_closed(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="receipt", source_path="receipt.jsonl", acquired_at_ms=1) + real_receipt = raw_authority_mod.raw_replay_application_receipt + + def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: + payload = dict(real_receipt(root, plan)) + payload["head_rows"] = [] + return json_document(payload) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=incomplete_receipt): + result = repair_raw_materialization(_config(tmp_path)) + + assert result.plan_outcomes[0].status is RawReplayPlanStatus.REJECTED_STALE + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + + +def test_stale_blocker_resolution_replans_current_evidence_and_resumes(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="resume", source_path="resume.jsonl", acquired_at_ms=1) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={"test": "resolve"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET source_path = 'resume-moved.jsonl' WHERE raw_id = ?", (raw_id,)) + conn.commit() + valid, observed = validate_raw_replay_plan(tmp_path, plan) + assert valid is False + rejected = reject_stale_raw_replay_plan(tmp_path, census.census_id, plan, observed) + blocker_id = cast(str, cast(dict[str, object], rejected.application_receipt)["blocker_id"]) + + resolution = resolve_raw_authority_blocker(tmp_path, blocker_id, resolution="current path is authoritative") + resumed = repair_raw_materialization(_config(tmp_path)) + + assert resolution["blocker_id"] == blocker_id + assert resumed.metrics.get("raw_materialization_unresolved_blocker_count", 0.0) == 0.0 + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 0 + ) + + +def test_fixed_point_compares_residual_identity_and_parser_fingerprint(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + first = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["a"]}, + ) + second = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["b"]}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + "UPDATE raw_authority_censuses SET parser_fingerprint = 'stale-parser' WHERE census_id = ?", + (second.census_id,), + ) + conn.commit() + third = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["b"]}, + ) + fourth = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["b"]}, + ) + assert first.fixed_point is False + assert second.fixed_point is False + assert third.fixed_point is False + assert fourth.fixed_point is True + + +def test_stale_per_raw_parser_fingerprint_is_recensused_before_planning(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="parser-drift", source_path="parser-drift.jsonl", acquired_at_ms=1) + first = repair_raw_materialization(_config(tmp_path), dry_run=True) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + "UPDATE raw_authority_parser_census SET parser_fingerprint = 'old-parser' WHERE raw_id = ?", + (raw_id,), + ) + conn.commit() + + second = repair_raw_materialization(_config(tmp_path), dry_run=True) + + assert first.plan_outcomes[0].plan_id == second.plan_outcomes[0].plan_id + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute( + "SELECT parser_fingerprint FROM raw_authority_parser_census WHERE raw_id = ?", + (raw_id,), + ).fetchone()[0] + == "revision-membership-v1" + ) + + +def test_repair_result_bounds_public_plan_outcomes() -> None: + outcomes = tuple( + RawReplayPlanOutcome( + f"plan-{index}", + (f"raw-{index}",), + RawReplayPlanStatus.RETRYABLE, + "test", + "retry", + ) + for index in range(10) + ) + result = RepairResult( + "raw_materialization", + MaintenanceCategory.DERIVED_REPAIR, + False, + 0, + False, + plan_outcomes=outcomes, + ).to_dict() + assert result["plan_outcome_count"] == 10 + assert len(cast(list[object], result["plan_outcomes"])) == 8 + assert result["plan_outcomes_truncated"] is True diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 718b350c8b..c8689f54ba 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -11,6 +11,7 @@ from polylogue.config import Config from polylogue.maintenance.models import DerivedModelStatus +from polylogue.sources.revision_backfill import census_historical_revision_evidence from polylogue.storage import repair as repair_mod from polylogue.storage.blob_store import BlobStore from polylogue.storage.insights.session.repair_assessment import assess_session_insight_repairs @@ -1636,7 +1637,7 @@ def __init__(self, **_kwargs: object) -> None: assert result.success is False assert result.repaired_count == 0 - assert "1 replay candidate(s) remain" in result.detail + assert "planning paused" in result.detail assert result.metrics["raw_materialization_oversized_count"] == 1.0 assert result.metrics["raw_materialization_resource_blocked_count"] == 1.0 assert result.metrics["raw_materialization_executed_count"] == 0.0 @@ -1663,6 +1664,7 @@ def test_raw_materialization_classifies_oversized_stream_record_replay( with sqlite3.connect(tmp_path / "source.db") as conn: conn.execute("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", (oversized, raw_id)) conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) config = _config(tmp_path) calls: dict[str, object] = {} @@ -1738,6 +1740,7 @@ def test_raw_materialization_blocks_oversized_expanded_cohort_before_blob_open( (repair_mod.RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + 1, oversized_raw), ) source_conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=[small_raw, oversized_raw]) monkeypatch.setattr( "polylogue.sources.revision_backfill._parse_retained_raw", @@ -1785,6 +1788,7 @@ def test_raw_materialization_backlog_expands_to_oversized_materialized_sibling( ("large-done", "codex-session", large_raw, "done", bytes(32)), ) index_conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=[small_raw, large_raw]) backlog = repair_mod.raw_materialization_replay_backlog(_config(tmp_path)) assert backlog["candidate_count"] == 1 @@ -1827,6 +1831,7 @@ def test_raw_materialization_blocks_aggregate_sub_limit_cohort_before_blob_open( ((per_raw_size, raw_id) for raw_id in raw_ids), ) source_conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=raw_ids) backlog = repair_mod.raw_materialization_replay_backlog(_config(tmp_path)) assert backlog["oversized_count"] == 0 @@ -1916,6 +1921,7 @@ def test_raw_materialization_durable_ledger_survives_ops_reset_for_fairness(tmp_ (repair_mod.RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + 1, raw_ids[0]), ) conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=raw_ids) first = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) assert first.plan_outcomes[0].status.value == "retryable" @@ -2072,7 +2078,7 @@ def __init__(self, **_kwargs: object) -> None: assert result.success is False assert result.repaired_count == 0 - assert "typed revision authority" in result.detail + assert "parser census completes" in result.detail assert result.metrics["raw_materialization_already_parsed_count"] == 1.0 From 84222717c536ebed56e3c996c3c064fe0a92cd66 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 16 Jul 2026 23:46:47 +0200 Subject: [PATCH 06/13] fix(storage): prove raw replay postflight conservation Persist the complete post-pass plan inventory before finalizing a raw-authority census. Exact recovery now rejects partial component memberships, exception paths inspect durable postconditions, repeated stale occurrences create fresh blockers, and operator resolution holds source/index writer exclusion. Bound blocker projection to the current census page and project the postflight digests through readiness and daemon receipts. Ref polylogue-hjpx.1. Ref polylogue-lkrc. --- docs/maintenance.md | 21 +- polylogue/daemon/cli.py | 3 + polylogue/storage/archive_readiness.py | 6 +- polylogue/storage/raw_authority.py | 270 ++++++++++++++---- polylogue/storage/repair.py | 120 +++++++- .../storage/sqlite/archive_tiers/source.py | 22 ++ .../source/013_raw_authority_ledger.sql | 22 ++ tests/unit/daemon/test_daemon_cli.py | 6 + tests/unit/storage/test_durable_migrations.py | 2 + .../unit/storage/test_raw_authority_ledger.py | 105 ++++++- 10 files changed, 507 insertions(+), 70 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index 327340887c..cc1c98f735 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -347,11 +347,13 @@ polylogue ops maintenance raw-authority-census \ --output-format json ``` -The response includes the census digests, complete witnesses and outcome for -the current page, any linked stale-plan blockers, and `next_query_handle` when -more rows remain. `--limit` is bounded to 1–500; `--offset` can override the -offset encoded in the URI. MCP clients resolve the URI directly through the -matching resource template. +The response includes both the before-plan inventory and the durable +postflight plan inventory/digests, complete witnesses and outcome for the +current page, and only the blockers created by that census page. +`next_query_handle` advances across the larger inventory without emitting +unbounded blocker history. `--limit` is bounded to 1–500; `--offset` can +override the offset encoded in the URI. MCP clients resolve the URI directly +through the matching resource template. Every receipt identifies its `mode` (`census`, `dry_run`, or `apply`), whether the parser census was `quiescent`, and its lifecycle. Apply receipts remain @@ -359,7 +361,14 @@ the parser census was `quiescent`, and its lifecycle. Apply receipts remain then validates exact source, application/membership, accepted-head, and session postconditions before marking an interrupted pass `executed`. Readiness never reports a `planned` row as the latest completed census and exposes its pending -count separately. +count separately. Finalization also proves that every retryable or +carried-forward plan has the identical immutable ID in the postflight census; +a partially applied component cannot be mislabeled as unchanged work. + +Raw-authority preview is the narrow exception to the generic read-only preview +rule above: it may durably record source-tier parser/census observations so a +moved-path component has one crash-safe identity across preview and apply. It +never selects or applies an index replay plan. A stale precondition or incomplete application receipt creates a durable, fail-closed blocker. After inspecting the census URI and current evidence, diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 00b835b998..60ad9882ff 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -652,6 +652,9 @@ def _emit_raw_materialization_pass(result: Any) -> None: "inventory_digest": census.inventory_digest, "residual_digest": census.residual_digest, "plan_count": census.plan_count, + "post_inventory_digest": census.post_inventory_digest, + "post_residual_digest": census.post_residual_digest, + "post_plan_count": census.post_plan_count, "executable_plan_count": census.executable_plan_count, "residual_plan_count": census.residual_plan_count, "predecessor_census_id": census.predecessor_census_id, diff --git a/polylogue/storage/archive_readiness.py b/polylogue/storage/archive_readiness.py index 3ec01f675f..1a457a569c 100644 --- a/polylogue/storage/archive_readiness.py +++ b/polylogue/storage/archive_readiness.py @@ -242,7 +242,8 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob census_row = conn.execute( """ SELECT census_id, sequence_no, inventory_digest, residual_digest, - plan_count, executable_plan_count, residual_plan_count, + plan_count, post_inventory_digest, post_residual_digest, + post_plan_count, executable_plan_count, residual_plan_count, predecessor_census_id, mode, lifecycle_status, quiescent, fixed_point, completed_at_ms FROM source.raw_authority_censuses @@ -257,6 +258,9 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "inventory_digest": str(census_row["inventory_digest"]), "residual_digest": str(census_row["residual_digest"]), "plan_count": int(census_row["plan_count"]), + "post_inventory_digest": str(census_row["post_inventory_digest"]), + "post_residual_digest": str(census_row["post_residual_digest"]), + "post_plan_count": int(census_row["post_plan_count"]), "executable_plan_count": int(census_row["executable_plan_count"]), "residual_plan_count": int(census_row["residual_plan_count"]), "predecessor_census_id": census_row["predecessor_census_id"], diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 862eea0d64..6b51649e1e 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -90,6 +90,9 @@ class RawAuthorityCensusReceipt: plan_count: int executable_plan_count: int residual_plan_count: int + post_inventory_digest: str | None + post_residual_digest: str | None + post_plan_count: int | None predecessor_census_id: str | None mode: str lifecycle_status: str @@ -157,6 +160,8 @@ def read_raw_authority_census( SELECT census_id, sequence_no, scope_json, residual_json, parser_fingerprint, mode, lifecycle_status, quiescent, inventory_digest, residual_digest, plan_count, + post_inventory_digest, post_residual_json, + post_residual_digest, post_plan_count, postflight_at_ms, executable_plan_count, residual_plan_count, predecessor_census_id, fixed_point, created_at_ms, completed_at_ms @@ -183,6 +188,18 @@ def read_raw_authority_census( """, (census_id, limit, resolved_offset), ).fetchall() + post_rows = conn.execute( + """ + SELECT cpp.ordinal, p.plan_id, p.input_digest, + p.input_raw_ids_json, p.logical_keys_json + FROM raw_authority_census_post_plans AS cpp + JOIN raw_authority_plans AS p ON p.plan_id = cpp.plan_id + WHERE cpp.census_id = ? + ORDER BY cpp.ordinal + LIMIT ? OFFSET ? + """, + (census_id, limit, resolved_offset), + ).fetchall() plan_ids = [str(row["plan_id"]) for row in rows] blockers: list[dict[str, object]] = [] if plan_ids: @@ -203,10 +220,10 @@ def read_raw_authority_census( SELECT blocker_id, plan_id, reason, expected_json, observed_json, created_at_ms, resolved_at_ms, resolution FROM raw_authority_blockers - WHERE plan_id IN ({marks}) + WHERE census_id = ? AND plan_id IN ({marks}) ORDER BY created_at_ms, blocker_id """, - plan_ids, + (census_id, *plan_ids), ) ] plans = [ @@ -233,12 +250,15 @@ def read_raw_authority_census( for row in rows ] total = int(census["plan_count"]) - next_offset = resolved_offset + len(plans) + post_total = int(census["post_plan_count"] or 0) + next_offset = resolved_offset + max(len(plans), len(post_rows)) return json_document( { "query_handle": raw_authority_census_query_handle(census_id, offset=resolved_offset), "next_query_handle": ( - raw_authority_census_query_handle(census_id, offset=next_offset) if next_offset < total else None + raw_authority_census_query_handle(census_id, offset=next_offset) + if next_offset < max(total, post_total) + else None ), "offset": resolved_offset, "limit": limit, @@ -255,6 +275,15 @@ def read_raw_authority_census( "inventory_digest": str(census["inventory_digest"]), "residual_digest": str(census["residual_digest"]), "plan_count": total, + "post_inventory_digest": census["post_inventory_digest"], + "post_residual": ( + _decode_json_field(census["post_residual_json"]) + if census["post_residual_json"] is not None + else None + ), + "post_residual_digest": census["post_residual_digest"], + "post_plan_count": post_total, + "postflight_at_ms": census["postflight_at_ms"], "executable_plan_count": int(census["executable_plan_count"]), "residual_plan_count": int(census["residual_plan_count"]), "predecessor_census_id": census["predecessor_census_id"], @@ -263,6 +292,16 @@ def read_raw_authority_census( "completed_at_ms": (int(census["completed_at_ms"]) if census["completed_at_ms"] is not None else None), }, "plans": plans, + "post_plans": [ + { + "ordinal": int(row["ordinal"]), + "plan_id": str(row["plan_id"]), + "input_digest": str(row["input_digest"]), + "input_raw_ids": _decode_json_field(row["input_raw_ids_json"]), + "logical_keys": _decode_json_field(row["logical_keys_json"]), + } + for row in post_rows + ], "blockers": blockers, } ) @@ -509,6 +548,11 @@ def record_raw_authority_census( census_id = f"census:{sequence_no}:{inventory_digest[:16]}:{residual_digest[:16]}" lifecycle_status = "planned" if mode == "apply" and selected_plan_ids else "completed" completed_at_ms = None if lifecycle_status == "planned" else now + post_inventory_digest = None if lifecycle_status == "planned" else inventory_digest + post_residual_json = None if lifecycle_status == "planned" else residual_json + post_residual_digest = None if lifecycle_status == "planned" else residual_digest + post_plan_count = None if lifecycle_status == "planned" else len(plans) + postflight_at_ms = None if lifecycle_status == "planned" else now for plan in plans: values = ( plan.plan_id, @@ -548,9 +592,11 @@ def record_raw_authority_census( census_id, sequence_no, scope_json, residual_json, parser_fingerprint, mode, lifecycle_status, quiescent, inventory_digest, residual_digest, plan_count, + post_inventory_digest, post_residual_json, + post_residual_digest, post_plan_count, postflight_at_ms, executable_plan_count, residual_plan_count, predecessor_census_id, fixed_point, created_at_ms, completed_at_ms - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( census_id, @@ -564,6 +610,11 @@ def record_raw_authority_census( inventory_digest, residual_digest, len(plans), + post_inventory_digest, + post_residual_json, + post_residual_digest, + post_plan_count, + postflight_at_ms, executable_count, residual_count, predecessor, @@ -596,6 +647,14 @@ def record_raw_authority_census( now, ), ) + if lifecycle_status != "planned": + conn.executemany( + """ + INSERT INTO raw_authority_census_post_plans (census_id, plan_id, ordinal) + VALUES (?, ?, ?) + """, + ((census_id, plan.plan_id, ordinal) for ordinal, plan in enumerate(plans)), + ) return RawAuthorityCensusReceipt( census_id=census_id, sequence_no=sequence_no, @@ -604,6 +663,9 @@ def record_raw_authority_census( plan_count=len(plans), executable_plan_count=executable_count, residual_plan_count=residual_count, + post_inventory_digest=post_inventory_digest, + post_residual_digest=post_residual_digest, + post_plan_count=post_plan_count, predecessor_census_id=predecessor, mode=mode, lifecycle_status=lifecycle_status, @@ -727,17 +789,28 @@ def rows(name: str) -> list[Mapping[str, object]]: problems.append("executed replay plan has no logical authority keys") elif head_keys != expected_keys: problems.append("accepted head keys do not match the immutable plan") - terminal_keys = { - str(row.get("logical_source_key")) - for row in application_rows - if row.get("decision") in {"selected_baseline", "applied_append", "superseded"} - } | { - str(row.get("logical_source_key")) - for row in membership_rows - if row.get("decision") in {"applied", "superseded_equivalent", "superseded_prefix"} - } - if not expected_keys.issubset(terminal_keys): - problems.append("not every logical key has a terminal application or membership decision") + input_raw_ids = set(plan.input_raw_ids) + witness = plan.authority_witness.get("memberships") + expected_memberships = ( + {(str(row.get("raw_id")), str(row.get("logical_source_key"))) for row in witness if isinstance(row, dict)} + if isinstance(witness, list) + else set() + ) + observed_memberships = {(str(row.get("raw_id")), str(row.get("logical_source_key"))) for row in membership_rows} + if observed_memberships != expected_memberships: + problems.append("membership receipt pairs do not match the immutable authority witness") + terminal_membership_decisions = {"applied", "superseded_equivalent", "superseded_prefix"} + if any(row.get("decision") not in terminal_membership_decisions for row in membership_rows): + problems.append("membership receipt contains a non-terminal decision") + terminal_application_decisions = {"selected_baseline", "applied_append", "superseded"} + application_pairs = {(str(row.get("raw_id")), str(row.get("logical_source_key"))) for row in application_rows} + if any(row.get("decision") not in terminal_application_decisions for row in application_rows): + problems.append("application receipt contains a non-terminal decision") + if any(raw_id not in input_raw_ids or key not in expected_keys for raw_id, key in application_pairs): + problems.append("application receipt contains authority outside the immutable component") + terminal_keys = {key for _, key in observed_memberships | application_pairs} + if terminal_keys != expected_keys: + problems.append("terminal receipt keys do not exactly match the immutable plan") head_session_ids = {str(row.get("session_id")) for row in head_rows} session_ids = {str(row.get("session_id")) for row in session_rows} if head_session_ids != session_ids: @@ -747,7 +820,6 @@ def rows(name: str) -> list[Mapping[str, object]]: str(row.get("accepted_content_hash")) != session_content.get(str(row.get("session_id"))) for row in head_rows ): problems.append("accepted head content hashes do not match materialized sessions") - input_raw_ids = set(plan.input_raw_ids) if any(str(row.get("accepted_raw_id")) not in input_raw_ids for row in head_rows): problems.append("accepted heads do not point into the immutable input component") return not problems, tuple(problems) @@ -800,6 +872,7 @@ def _raw_authority_census_receipt(conn: sqlite3.Connection, census_id: str) -> R """ SELECT census_id, sequence_no, inventory_digest, residual_digest, plan_count, executable_plan_count, residual_plan_count, + post_inventory_digest, post_residual_digest, post_plan_count, predecessor_census_id, mode, lifecycle_status, quiescent, fixed_point FROM raw_authority_censuses WHERE census_id = ? @@ -816,11 +889,14 @@ def _raw_authority_census_receipt(conn: sqlite3.Connection, census_id: str) -> R plan_count=int(row[4]), executable_plan_count=int(row[5]), residual_plan_count=int(row[6]), - predecessor_census_id=str(row[7]) if row[7] is not None else None, - mode=str(row[8]), - lifecycle_status=str(row[9]), - quiescent=bool(row[10]), - fixed_point=bool(row[11]), + post_inventory_digest=str(row[7]) if row[7] is not None else None, + post_residual_digest=str(row[8]) if row[8] is not None else None, + post_plan_count=int(row[9]) if row[9] is not None else None, + predecessor_census_id=str(row[10]) if row[10] is not None else None, + mode=str(row[11]), + lifecycle_status=str(row[12]), + quiescent=bool(row[13]), + fixed_point=bool(row[14]), ) @@ -828,6 +904,8 @@ def finalize_raw_authority_census( archive_root: Path, census_id: str, *, + post_plans: Sequence[RawReplayPlan], + post_residual: Mapping[str, object], interrupted: bool = False, ) -> RawAuthorityCensusReceipt: """Publish a census only after every selected plan has a recorded outcome.""" @@ -852,22 +930,93 @@ def finalize_raw_authority_census( ) if pending: raise RuntimeError(f"raw authority census still has {pending} pending selected outcome(s)") + post_ids = {plan.plan_id for plan in post_plans} + persistent = { + str(row[0]) + for row in conn.execute( + """ + SELECT plan_id FROM raw_authority_census_plans + WHERE census_id = ? AND outcome_status IN ('retryable', 'carried_forward') + """, + (census_id,), + ) + } + if not persistent.issubset(post_ids): + raise RuntimeError( + f"raw authority postflight changed a retryable/carried-forward plan: {sorted(persistent - post_ids)}" + ) + for plan in post_plans: + values = ( + plan.plan_id, + plan.input_digest, + _canonical_json(list(plan.input_raw_ids)), + _canonical_json(list(plan.logical_keys)), + _canonical_json(plan.authority_witness), + _canonical_json(plan.source_preconditions), + _canonical_json(plan.index_preconditions), + now, + ) + conn.execute( + """ + INSERT INTO raw_authority_plans ( + plan_id, input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(plan_id) DO NOTHING + """, + values, + ) + stored = conn.execute( + """ + SELECT input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json + FROM raw_authority_plans WHERE plan_id = ? + """, + (plan.plan_id,), + ).fetchone() + if stored != values[1:7]: + raise RuntimeError(f"immutable raw replay postflight plan collision: {plan.plan_id}") + conn.executemany( + """ + INSERT INTO raw_authority_census_post_plans (census_id, plan_id, ordinal) + VALUES (?, ?, ?) + """, + ((census_id, plan.plan_id, ordinal) for ordinal, plan in enumerate(post_plans)), + ) + post_inventory_digest = _digest([plan.plan_id for plan in post_plans]) + post_residual_json = _canonical_json(post_residual) + post_residual_digest = _digest(post_residual) conn.execute( """ UPDATE raw_authority_censuses - SET lifecycle_status = ?, completed_at_ms = ? + SET lifecycle_status = ?, completed_at_ms = ?, + post_inventory_digest = ?, post_residual_json = ?, + post_residual_digest = ?, post_plan_count = ?, postflight_at_ms = ? WHERE census_id = ? AND lifecycle_status = 'planned' """, - ("interrupted" if interrupted else "completed", now, census_id), + ( + "interrupted" if interrupted else "completed", + now, + post_inventory_digest, + post_residual_json, + post_residual_digest, + len(post_plans), + now, + census_id, + ), ) return _raw_authority_census_receipt(conn, census_id) -def recover_interrupted_raw_authority_censuses(archive_root: Path) -> int: +def recover_interrupted_raw_authority_censuses( + archive_root: Path, +) -> tuple[tuple[str, JSONDocument], ...]: """Reconcile unfinished apply censuses from durable postconditions.""" source_db = archive_root / "source.db" if not source_db.is_file(): - return 0 + return () with closing(sqlite3.connect(source_db)) as conn: conn.row_factory = sqlite3.Row rows = conn.execute( @@ -881,10 +1030,15 @@ def recover_interrupted_raw_authority_censuses(archive_root: Path) -> int: ORDER BY c.sequence_no, cp.ordinal """ ).fetchall() - census_ids = tuple( - str(row[0]) + census_scopes = tuple( + (str(row[0]), json_document(json.loads(str(row[1])))) for row in conn.execute( - "SELECT census_id FROM raw_authority_censuses WHERE lifecycle_status = 'planned' ORDER BY sequence_no" + """ + SELECT census_id, scope_json + FROM raw_authority_censuses + WHERE lifecycle_status = 'planned' + ORDER BY sequence_no + """ ) ) for row in rows: @@ -916,9 +1070,7 @@ def recover_interrupted_raw_authority_censuses(archive_root: Path) -> int: receipt, ) record_raw_replay_outcome(archive_root, census_id, outcome) - for census_id in census_ids: - finalize_raw_authority_census(archive_root, census_id, interrupted=True) - return len(census_ids) + return census_scopes def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolution: str) -> JSONDocument: @@ -926,8 +1078,10 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu if not resolution.strip(): raise ValueError("raw authority blocker resolution must be non-empty") source_db = archive_root / "source.db" - with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + with closing(sqlite3.connect(source_db)) as conn: conn.row_factory = sqlite3.Row + conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("BEGIN IMMEDIATE") row = conn.execute( """ SELECT b.blocker_id, b.plan_id, b.expected_json, p.input_raw_ids_json @@ -937,22 +1091,22 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu """, (blocker_id,), ).fetchone() - if row is None: - raise KeyError(blocker_id) - input_raw_ids = tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))) - observed = build_raw_replay_plans(archive_root, (input_raw_ids,))[0] - now = int(time.time() * 1000) - receipt = json_document( - { - "schema": "polylogue.raw-authority-blocker-resolution.v1", - "blocker_id": blocker_id, - "superseded_plan_id": str(row["plan_id"]), - "current_plan": observed.to_dict(), - "operator_resolution": resolution.strip(), - "resolved_at_ms": now, - } - ) - with closing(sqlite3.connect(source_db)) as conn, conn: + if row is None: + conn.rollback() + raise KeyError(blocker_id) + input_raw_ids = tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))) + observed = build_raw_replay_plan(conn, input_raw_ids) + now = int(time.time() * 1000) + receipt = json_document( + { + "schema": "polylogue.raw-authority-blocker-resolution.v1", + "blocker_id": blocker_id, + "superseded_plan_id": str(row["plan_id"]), + "current_plan": observed.to_dict(), + "operator_resolution": resolution.strip(), + "resolved_at_ms": now, + } + ) updated = conn.execute( """ UPDATE raw_authority_blockers @@ -962,7 +1116,9 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu (now, _canonical_json(receipt), blocker_id), ).rowcount if updated != 1: + conn.rollback() raise RuntimeError(f"raw authority blocker changed during resolution: {blocker_id}") + conn.commit() return receipt @@ -974,7 +1130,7 @@ def reject_stale_raw_replay_plan( ) -> RawReplayPlanOutcome: """Persist the fail-closed blocker before returning observational output.""" now = int(time.time() * 1000) - blocker_id = f"raw-authority-blocker:{_digest([plan.plan_id, observed])}" + blocker_id = f"raw-authority-blocker:{_digest([census_id, plan.plan_id, observed])}" outcome = RawReplayPlanOutcome( plan.plan_id, plan.input_raw_ids, @@ -1022,6 +1178,12 @@ def reject_stale_raw_replay_plan( ).rowcount if updated != 1: raise RuntimeError(f"stale rejection does not conserve one selected plan: {plan.plan_id}") + open_count = conn.execute( + "SELECT COUNT(*) FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (plan.plan_id,), + ).fetchone()[0] + if int(open_count) != 1: + raise RuntimeError(f"stale rejection did not leave one open blocker: {plan.plan_id}") return outcome @@ -1035,7 +1197,7 @@ def reject_invalid_raw_replay_application( """Fail closed when a writer returns without exact application postconditions.""" now = int(time.time() * 1000) observed = json_document({"application_receipt": receipt, "problems": list(problems)}) - blocker_id = f"raw-authority-blocker:{_digest([plan.plan_id, observed])}" + blocker_id = f"raw-authority-blocker:{_digest([census_id, plan.plan_id, observed])}" outcome = RawReplayPlanOutcome( plan.plan_id, plan.input_raw_ids, @@ -1083,6 +1245,12 @@ def reject_invalid_raw_replay_application( ).rowcount if updated != 1: raise RuntimeError(f"invalid application does not conserve one selected plan: {plan.plan_id}") + open_count = conn.execute( + "SELECT COUNT(*) FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (plan.plan_id,), + ).fetchone()[0] + if int(open_count) != 1: + raise RuntimeError(f"invalid application did not leave one open blocker: {plan.plan_id}") return outcome diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 7250cc215e..9a965b6f0e 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -4986,6 +4986,38 @@ def _raw_authority_residual( } +def _raw_authority_postflight_snapshot( + archive_root: Path, + candidates: RawMaterializationCandidates, +) -> tuple[tuple[RawReplayPlan, ...], dict[str, object]]: + """Build the complete post-pass plan inventory and typed residual debt.""" + components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + plans = build_raw_replay_plans(archive_root, components) + blocked_plan_ids = tuple( + sorted( + plan.plan_id + for component, plan in zip(components, plans, strict=True) + if sum(_raw_materialization_component_blob_bytes(candidates, member) for member in component) + > RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + ) + ) + return plans, _raw_authority_residual(candidates, resource_blocked_plan_ids=blocked_plan_ids) + + +def _raw_authority_candidates_for_scope( + config: Config, + scope: Mapping[str, object], +) -> RawMaterializationCandidates: + source_root_value = scope.get("source_root") + return _raw_materialization_candidate_ids( + config, + raw_artifact_id=str(scope["raw_artifact_id"]) if scope.get("raw_artifact_id") is not None else None, + provider=str(scope["provider"]) if scope.get("provider") is not None else None, + source_family=str(scope["source_family"]) if scope.get("source_family") is not None else None, + source_root=Path(str(source_root_value)) if source_root_value is not None else None, + ) + + def _raw_materialization_base_metrics( candidates: RawMaterializationCandidates, *, @@ -5622,6 +5654,9 @@ def to_dict(self) -> JSONDocument: "inventory_digest": self.census_receipt.inventory_digest, "residual_digest": self.census_receipt.residual_digest, "plan_count": self.census_receipt.plan_count, + "post_inventory_digest": self.census_receipt.post_inventory_digest, + "post_residual_digest": self.census_receipt.post_residual_digest, + "post_plan_count": self.census_receipt.post_plan_count, "executable_plan_count": self.census_receipt.executable_plan_count, "residual_plan_count": self.census_receipt.residual_plan_count, "predecessor_census_id": self.census_receipt.predecessor_census_id, @@ -6447,7 +6482,21 @@ def repair_raw_materialization( ) -> RepairResult: """Converge retained raws through typed per-session revision authority.""" archive_root = _raw_materialization_archive_root(config) - recovered_census_count = recover_interrupted_raw_authority_censuses(archive_root) + recovered_censuses = recover_interrupted_raw_authority_censuses(archive_root) + for recovered_census_id, recovered_scope in recovered_censuses: + recovered_candidates = _raw_authority_candidates_for_scope(config, recovered_scope) + recovered_post_plans, recovered_post_residual = _raw_authority_postflight_snapshot( + archive_root, + recovered_candidates, + ) + finalize_raw_authority_census( + archive_root, + recovered_census_id, + post_plans=recovered_post_plans, + post_residual=recovered_post_residual, + interrupted=True, + ) + recovered_census_count = len(recovered_censuses) blocker_count = unresolved_raw_authority_blockers(archive_root) if blocker_count: return _internal_derived_repair_result( @@ -6793,7 +6842,23 @@ def repair_raw_materialization( ] for outcome in carried: record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) - census_receipt = finalize_raw_authority_census(archive_root, census_receipt.census_id) + stale_candidates = _raw_materialization_candidate_ids( + config, + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + ) + stale_post_plans, stale_post_residual = _raw_authority_postflight_snapshot( + archive_root, + stale_candidates, + ) + census_receipt = finalize_raw_authority_census( + archive_root, + census_receipt.census_id, + post_plans=stale_post_plans, + post_residual=stale_post_residual, + ) plan_outcomes = tuple(stale_outcomes + carried) + blocked_plan_outcomes metrics["raw_materialization_plan_rejected_stale_count"] = float(len(stale_outcomes)) metrics["raw_materialization_plan_carried_forward_count"] = float(len(carried)) @@ -6842,14 +6907,41 @@ def repair_raw_materialization( continue except Exception as exc: logger.exception("raw replay plan %s failed", plan.plan_id) - outcome = RawReplayPlanOutcome( - plan.plan_id, - component, - RawReplayPlanStatus.RETRYABLE, - f"component execution raised {type(exc).__name__}: {exc}", - "retry this plan after independent components have received a turn", - ) - record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + application_receipt = raw_replay_application_receipt(archive_root, plan) + receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) + if receipt_valid: + outcome = RawReplayPlanOutcome( + plan.plan_id, + component, + RawReplayPlanStatus.EXECUTED, + f"component reached exact durable postconditions before {type(exc).__name__}", + "none", + application_receipt, + ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + else: + plan_still_valid, _ = validate_raw_replay_plan(archive_root, plan) + if plan_still_valid: + outcome = RawReplayPlanOutcome( + plan.plan_id, + component, + RawReplayPlanStatus.RETRYABLE, + f"component execution raised {type(exc).__name__}: {exc}", + "retry this unchanged plan after independent components have received a turn", + application_receipt, + ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + else: + outcome = reject_invalid_raw_replay_application( + archive_root, + census_receipt.census_id, + plan, + application_receipt, + ( + f"component execution raised {type(exc).__name__}: {exc}", + *receipt_problems, + ), + ) execution_outcomes.append(outcome) continue replay_parts.append(part) @@ -6919,7 +7011,13 @@ def repair_raw_materialization( } ) plan_outcomes = tuple(execution_outcomes) + blocked_plan_outcomes - census_receipt = finalize_raw_authority_census(archive_root, census_receipt.census_id) + post_plans, post_residual = _raw_authority_postflight_snapshot(archive_root, remaining) + census_receipt = finalize_raw_authority_census( + archive_root, + census_receipt.census_id, + post_plans=post_plans, + post_residual=post_residual, + ) for status in RawReplayPlanStatus: metrics[f"raw_materialization_plan_{status.value}_count"] = float( sum(outcome.status is status for outcome in plan_outcomes) diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index d1728660fd..f656c355d7 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -125,6 +125,11 @@ inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), plan_count INTEGER NOT NULL CHECK(plan_count >= 0), + post_inventory_digest TEXT CHECK(post_inventory_digest IS NULL OR length(post_inventory_digest) = 64), + post_residual_json TEXT CHECK(post_residual_json IS NULL OR json_valid(post_residual_json)), + post_residual_digest TEXT CHECK(post_residual_digest IS NULL OR length(post_residual_digest) = 64), + post_plan_count INTEGER CHECK(post_plan_count IS NULL OR post_plan_count >= 0), + postflight_at_ms INTEGER CHECK(postflight_at_ms IS NULL OR postflight_at_ms >= created_at_ms), executable_plan_count INTEGER NOT NULL CHECK(executable_plan_count >= 0), residual_plan_count INTEGER NOT NULL CHECK(residual_plan_count >= 0), predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), @@ -136,6 +141,15 @@ CHECK( (lifecycle_status = 'planned' AND completed_at_ms IS NULL) OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) + ), + CHECK( + (lifecycle_status = 'planned' AND post_inventory_digest IS NULL + AND post_residual_json IS NULL AND post_residual_digest IS NULL + AND post_plan_count IS NULL AND postflight_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') + AND post_inventory_digest IS NOT NULL AND post_residual_json IS NOT NULL + AND post_residual_digest IS NOT NULL AND post_plan_count IS NOT NULL + AND postflight_at_ms IS NOT NULL) ) ) STRICT; @@ -175,6 +189,14 @@ ON raw_authority_census_plans(plan_id, recorded_at_ms DESC) WHERE selected = 1; +CREATE TABLE IF NOT EXISTS raw_authority_census_post_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + CREATE TABLE IF NOT EXISTS raw_authority_blockers ( blocker_id TEXT PRIMARY KEY, plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), diff --git a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql index 908eb76eb3..4b849b2085 100644 --- a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql +++ b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql @@ -21,6 +21,11 @@ CREATE TABLE raw_authority_censuses ( inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), plan_count INTEGER NOT NULL CHECK(plan_count >= 0), + post_inventory_digest TEXT CHECK(post_inventory_digest IS NULL OR length(post_inventory_digest) = 64), + post_residual_json TEXT CHECK(post_residual_json IS NULL OR json_valid(post_residual_json)), + post_residual_digest TEXT CHECK(post_residual_digest IS NULL OR length(post_residual_digest) = 64), + post_plan_count INTEGER CHECK(post_plan_count IS NULL OR post_plan_count >= 0), + postflight_at_ms INTEGER CHECK(postflight_at_ms IS NULL OR postflight_at_ms >= created_at_ms), executable_plan_count INTEGER NOT NULL CHECK(executable_plan_count >= 0), residual_plan_count INTEGER NOT NULL CHECK(residual_plan_count >= 0), predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), @@ -32,6 +37,15 @@ CREATE TABLE raw_authority_censuses ( CHECK( (lifecycle_status = 'planned' AND completed_at_ms IS NULL) OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) + ), + CHECK( + (lifecycle_status = 'planned' AND post_inventory_digest IS NULL + AND post_residual_json IS NULL AND post_residual_digest IS NULL + AND post_plan_count IS NULL AND postflight_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') + AND post_inventory_digest IS NOT NULL AND post_residual_json IS NOT NULL + AND post_residual_digest IS NOT NULL AND post_plan_count IS NOT NULL + AND postflight_at_ms IS NOT NULL) ) ) STRICT; @@ -71,6 +85,14 @@ CREATE INDEX idx_raw_authority_census_plans_attempts ON raw_authority_census_plans(plan_id, recorded_at_ms DESC) WHERE selected = 1; +CREATE TABLE raw_authority_census_post_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + CREATE TABLE raw_authority_blockers ( blocker_id TEXT PRIMARY KEY, plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 92d6dd177f..11c12a39f0 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -688,6 +688,9 @@ def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: py inventory_digest="a" * 64, residual_digest="b" * 64, plan_count=3, + post_inventory_digest="c" * 64, + post_residual_digest="d" * 64, + post_plan_count=2, executable_plan_count=1, residual_plan_count=2, predecessor_census_id="census:1:inventory:residual", @@ -715,6 +718,9 @@ def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: py "inventory_digest": "a" * 64, "residual_digest": "b" * 64, "plan_count": 3, + "post_inventory_digest": "c" * 64, + "post_residual_digest": "d" * 64, + "post_plan_count": 2, "executable_plan_count": 1, "residual_plan_count": 2, "predecessor_census_id": census.predecessor_census_id, diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index f67ded3745..850d060b31 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -555,6 +555,7 @@ def test_source_publication_backfill_requires_verified_backup( with sqlite3.connect(db_path) as conn: conn.execute("DROP TABLE raw_authority_parser_census") conn.execute("DROP TABLE raw_authority_blockers") + conn.execute("DROP TABLE raw_authority_census_post_plans") conn.execute("DROP TABLE raw_authority_census_plans") conn.execute("DROP TABLE raw_authority_plans") conn.execute("DROP TABLE raw_authority_censuses") @@ -588,6 +589,7 @@ def test_source_publication_backfill_requires_verified_backup( "raw_authority_parser_census", "raw_authority_plans", "raw_authority_census_plans", + "raw_authority_census_post_plans", "raw_authority_blockers", } <= tables diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index d141cef5e8..dc641614e8 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -101,6 +101,9 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat assert result.census_receipt.plan_count == 2 assert result.census_receipt.executable_plan_count == 2 assert result.census_receipt.residual_plan_count == 0 + assert result.census_receipt.post_plan_count == 1 + assert result.census_receipt.post_inventory_digest is not None + assert result.census_receipt.post_inventory_digest != result.census_receipt.inventory_digest assert result.census_receipt.lifecycle_status == "completed" assert result.metrics["raw_materialization_plan_outcome_count"] == 2.0 assert result.metrics["raw_materialization_plan_carried_forward_count"] == 1.0 @@ -156,6 +159,8 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat cast(dict[str, object], item)["outcome_status"] for item in (*cast(list[object], first_page["plans"]), *cast(list[object], second_page["plans"])) } == {"executed", "carried_forward"} + assert cast(dict[str, object], first_page["census"])["post_plan_count"] == 1 + assert len(cast(list[object], first_page["post_plans"])) == 1 def test_two_successive_quiescent_censuses_are_required_for_fixed_point(tmp_path: Path) -> None: @@ -284,7 +289,13 @@ def test_interrupted_census_has_no_partial_plan_visibility_and_retries_once(tmp_ "retry", ), ) - finalized = finalize_raw_authority_census(tmp_path, receipt.census_id, interrupted=True) + finalized = finalize_raw_authority_census( + tmp_path, + receipt.census_id, + post_plans=plans, + post_residual={}, + interrupted=True, + ) assert finalized.lifecycle_status == "interrupted" @@ -370,6 +381,50 @@ def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: ) +def test_recovery_rejects_partial_expanded_membership_postconditions(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw( + tmp_path, + native_id="partial-component", + source_path="partial-old.jsonl", + acquired_at_ms=1, + text="old", + ) + second = _write_codex_raw( + tmp_path, + native_id="partial-component", + source_path="partial-new.jsonl", + acquired_at_ms=2, + text="new", + ) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=RuntimeError("synthetic crash")): + with pytest.raises(RuntimeError, match="synthetic crash"): + repair_raw_materialization(_config(tmp_path)) + + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("DELETE FROM raw_session_memberships WHERE raw_id = ?", (second,)) + conn.commit() + + recovered = repair_raw_materialization(_config(tmp_path)) + + assert recovered.success is False + assert recovered.metrics["raw_materialization_unresolved_blocker_count"] == 1.0 + with sqlite3.connect(tmp_path / "source.db") as conn: + row = conn.execute( + """ + SELECT cp.outcome_status + FROM raw_authority_census_plans AS cp + JOIN raw_authority_censuses AS c ON c.census_id = cp.census_id + WHERE c.lifecycle_status = 'interrupted' AND cp.selected = 1 + """ + ).fetchone() + assert row == ("rejected_stale",) + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + + def test_stale_blocker_resolution_replans_current_evidence_and_resumes(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) raw_id = _write_codex_raw(tmp_path, native_id="resume", source_path="resume.jsonl", acquired_at_ms=1) @@ -403,6 +458,54 @@ def test_stale_blocker_resolution_replans_current_evidence_and_resumes(tmp_path: ) +def test_identical_stale_rejection_after_resolution_creates_new_open_blocker(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="repeat", source_path="repeat.jsonl", acquired_at_ms=1) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET source_path = 'repeat-moved.jsonl' WHERE raw_id = ?", (raw_id,)) + conn.commit() + valid, observed = validate_raw_replay_plan(tmp_path, plan) + assert valid is False + + first_census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={"test": "repeat-1"}, + residual={}, + ) + first = reject_stale_raw_replay_plan(tmp_path, first_census.census_id, plan, observed) + first_blocker = cast(str, cast(dict[str, object], first.application_receipt)["blocker_id"]) + resolve_raw_authority_blocker(tmp_path, first_blocker, resolution="acknowledge first occurrence") + + second_census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={"test": "repeat-2"}, + residual={}, + ) + second = reject_stale_raw_replay_plan(tmp_path, second_census.census_id, plan, observed) + second_blocker = cast(str, cast(dict[str, object], second.application_receipt)["blocker_id"]) + + assert second_blocker != first_blocker + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + page = read_raw_authority_census(tmp_path, second_census.query_handle) + blockers = cast(list[object], page["blockers"]) + assert len(blockers) == 1 + assert cast(dict[str, object], blockers[0])["blocker_id"] == second_blocker + + def test_fixed_point_compares_residual_identity_and_parser_fingerprint(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) first = record_raw_authority_census( From eef5c14e96c0b45cff813e98888b1ac172ce7a51 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 16 Jul 2026 23:58:22 +0200 Subject: [PATCH 07/13] fix(storage): close raw authority ledger review gaps Require a verified backup for the durable source-tier migration and enforce exact plan partitioning in the schema. Persist parser-census evidence on the direct historical-backfill path, report conservation over the complete plan inventory, and keep raw-row readiness counts separate from ledger-record counts. Bound daemon and repair projections by omitting full application receipts and sampling raw identifiers. Strengthen recovery, blocker-resolution, migration, and projection regressions with production outcome types. Ref polylogue-hjpx.1. --- polylogue/daemon/cli.py | 2 +- polylogue/sources/revision_backfill.py | 7 +++ polylogue/storage/archive_readiness.py | 12 ++--- polylogue/storage/raw_authority.py | 16 +++++++ polylogue/storage/repair.py | 47 +++++++++++++++---- .../storage/sqlite/archive_tiers/source.py | 1 + .../source/013_raw_authority_ledger.sql | 2 +- tests/unit/daemon/test_daemon_cli.py | 27 ++++++----- tests/unit/sources/test_revision_backfill.py | 10 ++++ .../unit/storage/test_raw_authority_ledger.py | 41 ++++++++++++++-- 10 files changed, 133 insertions(+), 32 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 60ad9882ff..93f1554b5b 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -671,7 +671,7 @@ def _emit_raw_materialization_pass(result: Any) -> None: "detail": str(result.detail), "metrics": metrics, "plan_outcome_count": len(outcomes), - "plan_outcome_sample": [outcome.to_dict() for outcome in outcomes[:outcome_sample_limit]], + "plan_outcome_sample": [outcome.to_summary_dict() for outcome in outcomes[:outcome_sample_limit]], "plan_outcome_sample_truncated": len(outcomes) > outcome_sample_limit, } if census_payload is not None: diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 1bc876a7ac..c557b7f4a9 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -308,6 +308,13 @@ def backfill_historical_revision_evidence( selected_raw_ids=selected_raw_ids, max_payload_bytes=max_payload_bytes, ) + censused_raw_ids, _censused_keys = archive.expand_raw_membership_selection(selected_raw_ids) + # The direct backfill entry point must publish the same current-parser + # receipt as the census-only entry point before it assigns or applies + # any index plan. Commit the source census first so the separate + # durable receipt writer observes one complete source snapshot. + archive.commit() + _record_raw_authority_parser_census(archive_root, tuple(censused_raw_ids)) membership_candidates = census.membership_candidates provisional_full_raw_ids = census.provisional_full_raw_ids diff --git a/polylogue/storage/archive_readiness.py b/polylogue/storage/archive_readiness.py index 1a457a569c..5391cccc58 100644 --- a/polylogue/storage/archive_readiness.py +++ b/polylogue/storage/archive_readiness.py @@ -309,10 +309,6 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob } if adoption_deferred_count: category_counts["adoption_deferred"] = adoption_deferred_count - if authority_blocker_count: - category_counts["raw_authority_blocker"] = authority_blocker_count - if authority_pending_census_count: - category_counts["raw_authority_pending_census"] = authority_pending_census_count category_counts.update( {category: count for category, count in classified_counts.items() if category != "parse-failed"} ) @@ -328,12 +324,12 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "critical": critical, "warning": 0, "actionable": actionable, - "blocked": adoption_deferred_count + authority_blocker_count + authority_pending_census_count, + "blocked": adoption_deferred_count, "classified": classified, "unchecked": unchecked, "affected_total": total, "affected_actionable": affected_actionable, - "affected_blocked": adoption_deferred_count + authority_blocker_count + authority_pending_census_count, + "affected_blocked": adoption_deferred_count, "affected_open": 0, "affected_classified": classified, "affected_unchecked": unchecked, @@ -344,6 +340,10 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "raw_authority_census": authority_census, "raw_authority_blocker_count": authority_blocker_count, "raw_authority_pending_census_count": authority_pending_census_count, + "raw_authority_ledger_counts": { + "unresolved_blockers": authority_blocker_count, + "pending_censuses": authority_pending_census_count, + }, } diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 6b51649e1e..5a25dd889a 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -80,6 +80,22 @@ def to_dict(self) -> JSONDocument: payload["application_receipt"] = self.application_receipt return json_document(payload) + def to_summary_dict(self) -> JSONDocument: + """Return a bounded projection; the census handle owns full receipts.""" + raw_id_sample_limit = 8 + return json_document( + { + "plan_id": self.plan_id, + "input_raw_count": len(self.input_raw_ids), + "input_raw_id_sample": list(self.input_raw_ids[:raw_id_sample_limit]), + "input_raw_id_sample_truncated": len(self.input_raw_ids) > raw_id_sample_limit, + "status": self.status.value, + "reason": self.reason, + "next_action": self.next_action, + "has_application_receipt": self.application_receipt is not None, + } + ) + @dataclass(frozen=True, slots=True) class RawAuthorityCensusReceipt: diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 9a965b6f0e..8bf9536133 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -5153,6 +5153,27 @@ def _raw_replay_plan_outcomes( return tuple(_raw_replay_plan_outcome(conn, plan, remaining=remaining) for plan in plans) +def _raw_replay_conservation_metrics( + plans: Sequence[RawReplayPlan], + selected_plan_ids: set[str], + outcomes: Sequence[RawReplayPlanOutcome], +) -> tuple[int, int, int]: + """Return total plans, all carried inventory, and exact algebra errors.""" + outcome_ids = [outcome.plan_id for outcome in outcomes] + outcome_id_set = set(outcome_ids) + conservation_errors = ( + len(selected_plan_ids - outcome_id_set) + + len(outcome_id_set - selected_plan_ids) + + (len(outcome_ids) - len(outcome_id_set)) + ) + carried_forward = ( + len(plans) + - len(selected_plan_ids) + + sum(outcome.status is RawReplayPlanStatus.CARRIED_FORWARD for outcome in outcomes) + ) + return len(plans), carried_forward, conservation_errors + + def _raw_materialization_bucket_summary( candidates: RawMaterializationCandidates, *, @@ -5644,7 +5665,7 @@ def to_dict(self) -> JSONDocument: if self.plan_outcomes: payload["plan_outcome_count"] = len(self.plan_outcomes) payload["plan_outcomes"] = [ - outcome.to_dict() for outcome in self.plan_outcomes[:RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT] + outcome.to_summary_dict() for outcome in self.plan_outcomes[:RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT] ] payload["plan_outcomes_truncated"] = len(self.plan_outcomes) > RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT if self.census_receipt is not None: @@ -6860,10 +6881,15 @@ def repair_raw_materialization( post_residual=stale_post_residual, ) plan_outcomes = tuple(stale_outcomes + carried) + blocked_plan_outcomes + plan_count, carried_forward_count, conservation_error_count = _raw_replay_conservation_metrics( + plans, + selected_plan_ids, + plan_outcomes, + ) metrics["raw_materialization_plan_rejected_stale_count"] = float(len(stale_outcomes)) - metrics["raw_materialization_plan_carried_forward_count"] = float(len(carried)) - metrics["raw_materialization_plan_outcome_count"] = float(len(plan_outcomes)) - metrics["raw_materialization_plan_conservation_error_count"] = float(len(stale_outcomes)) + metrics["raw_materialization_plan_carried_forward_count"] = float(carried_forward_count) + metrics["raw_materialization_plan_outcome_count"] = float(plan_count) + metrics["raw_materialization_plan_conservation_error_count"] = float(conservation_error_count) return _internal_derived_repair_result( "raw_materialization", repaired_count=0, @@ -7022,13 +7048,14 @@ def repair_raw_materialization( metrics[f"raw_materialization_plan_{status.value}_count"] = float( sum(outcome.status is status for outcome in plan_outcomes) ) - carried_forward_count = len(plans) - len(selected_components) - metrics["raw_materialization_plan_carried_forward_count"] = float(carried_forward_count) - metrics["raw_materialization_plan_outcome_count"] = float(len(plans)) - metrics["raw_materialization_plan_conservation_error_count"] = float( - sum(outcome.status is RawReplayPlanStatus.REJECTED_STALE for outcome in plan_outcomes) - + abs(len(selected_components) - len(plan_outcomes)) + plan_count, carried_forward_count, conservation_error_count = _raw_replay_conservation_metrics( + plans, + selected_plan_ids, + plan_outcomes, ) + metrics["raw_materialization_plan_carried_forward_count"] = float(carried_forward_count) + metrics["raw_materialization_plan_outcome_count"] = float(plan_count) + metrics["raw_materialization_plan_conservation_error_count"] = float(conservation_error_count) success = ( not remaining.raw_ids and remaining.missing_blobs == 0 diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index f656c355d7..037161676b 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -138,6 +138,7 @@ completed_at_ms INTEGER CHECK(completed_at_ms IS NULL OR completed_at_ms >= created_at_ms), CHECK(plan_count >= executable_plan_count), CHECK(plan_count >= residual_plan_count), + CHECK(plan_count = executable_plan_count + residual_plan_count), CHECK( (lifecycle_status = 'planned' AND completed_at_ms IS NULL) OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) diff --git a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql index 4b849b2085..a5ec20a78b 100644 --- a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql +++ b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql @@ -1,4 +1,3 @@ --- migration-safety: additive-no-backup -- Durable, restart-safe conservation ledger for raw authority reconciliation. CREATE TABLE raw_authority_parser_census ( raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, @@ -34,6 +33,7 @@ CREATE TABLE raw_authority_censuses ( completed_at_ms INTEGER CHECK(completed_at_ms IS NULL OR completed_at_ms >= created_at_ms), CHECK(plan_count >= executable_plan_count), CHECK(plan_count >= residual_plan_count), + CHECK(plan_count = executable_plan_count + residual_plan_count), CHECK( (lifecycle_status = 'planned' AND completed_at_ms IS NULL) OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 11c12a39f0..5f9944ab51 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -21,6 +21,7 @@ from polylogue.daemon.health import DaemonHealth, HealthSeverity, HealthTier from polylogue.sources.live import WatchSource from polylogue.sources.live.cursor import CursorStore +from polylogue.storage.raw_authority import RawReplayPlanOutcome, RawReplayPlanStatus from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database from polylogue.storage.sqlite.archive_tiers.embeddings import EMBEDDINGS_SCHEMA_VERSION from polylogue.storage.sqlite.archive_tiers.index import INDEX_SCHEMA_VERSION @@ -571,17 +572,12 @@ def test_raw_materialization_pass_emits_conserved_plan_receipt(monkeypatch: pyte from polylogue.daemon import cli as daemon_cli events: list[tuple[str, dict[str, object]]] = [] - outcome = SimpleNamespace( - status=SimpleNamespace(value="executed"), + outcome = RawReplayPlanOutcome( plan_id="raw-replay:stable", + input_raw_ids=("raw-a",), + status=RawReplayPlanStatus.EXECUTED, reason="applied", - to_dict=lambda: { - "plan_id": "raw-replay:stable", - "input_raw_ids": ["raw-a"], - "status": "executed", - "reason": "applied", - "next_action": "none", - }, + next_action="none", ) monkeypatch.setattr( "polylogue.daemon.events.emit_daemon_event", @@ -615,7 +611,7 @@ def test_raw_materialization_pass_emits_conserved_plan_receipt(monkeypatch: pyte "raw_materialization_remaining_candidate_count": 0.0, }, "plan_outcome_count": 1, - "plan_outcome_sample": [outcome.to_dict()], + "plan_outcome_sample": [outcome.to_summary_dict()], "plan_outcome_sample_truncated": False, }, ) @@ -652,7 +648,16 @@ def test_raw_materialization_pass_bounds_outcome_sample(monkeypatch: pytest.Monk from polylogue.daemon import cli as daemon_cli events: list[tuple[str, dict[str, object]]] = [] - outcomes = tuple(SimpleNamespace(to_dict=lambda index=index: {"plan_id": f"plan:{index}"}) for index in range(9)) + outcomes = tuple( + RawReplayPlanOutcome( + plan_id=f"plan:{index}", + input_raw_ids=(f"raw:{index}",), + status=RawReplayPlanStatus.CARRIED_FORWARD, + reason="not selected in bounded pass", + next_action="retry later", + ) + for index in range(9) + ) monkeypatch.setattr( "polylogue.daemon.events.emit_daemon_event", lambda kind, *, payload: events.append((kind, payload)), diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 59ea6e21d5..f8036a9497 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -106,6 +106,16 @@ def test_historical_backfill_selects_prefix_newest_independent_of_acquisition_or assert result.classified_full == 2 assert result.replayed_logical_sources == 1 assert result.quarantined == 1 + with sqlite3.connect(tmp_path / "source.db") as conn: + parser_census = conn.execute( + """ + SELECT status, COUNT(*) + FROM raw_authority_parser_census + WHERE parser_fingerprint = 'revision-membership-v1' + GROUP BY status ORDER BY status + """ + ).fetchall() + assert parser_census == [("complete", 2), ("failed", 1)] with sqlite3.connect(tmp_path / "index.db") as conn: assert conn.execute("SELECT message_count, raw_id FROM sessions").fetchone() == (2, newest_raw_id) diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index dc641614e8..936446171e 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -352,13 +352,20 @@ def test_interrupted_apply_recovers_exact_durable_postconditions(tmp_path: Path) with sqlite3.connect(tmp_path / "source.db") as conn: row = conn.execute( """ - SELECT c.lifecycle_status, cp.outcome_status + SELECT c.lifecycle_status, cp.outcome_status, cp.application_receipt_json FROM raw_authority_censuses AS c JOIN raw_authority_census_plans AS cp ON cp.census_id = c.census_id WHERE c.lifecycle_status = 'interrupted' """ ).fetchone() - assert row == ("interrupted", "executed") + assert row is not None + assert row[:2] == ("interrupted", "executed") + recovered_receipt = json.loads(row[2]) + assert isinstance(recovered_receipt["application_rows"], list) + assert isinstance(recovered_receipt["membership_rows"], list) + assert recovered_receipt["application_rows"] or recovered_receipt["membership_rows"] + assert recovered_receipt["head_rows"] + assert recovered_receipt["session_rows"] def test_parsed_timestamp_without_exact_application_receipt_fails_closed(tmp_path: Path) -> None: @@ -451,7 +458,9 @@ def test_stale_blocker_resolution_replans_current_evidence_and_resumes(tmp_path: resumed = repair_raw_materialization(_config(tmp_path)) assert resolution["blocker_id"] == blocker_id - assert resumed.metrics.get("raw_materialization_unresolved_blocker_count", 0.0) == 0.0 + assert resumed.success is True + assert resumed.repaired_count == 1 + assert "raw_materialization_unresolved_blocker_count" not in resumed.metrics with sqlite3.connect(tmp_path / "source.db") as conn: assert ( conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 0 @@ -606,3 +615,29 @@ def test_repair_result_bounds_public_plan_outcomes() -> None: assert result["plan_outcome_count"] == 10 assert len(cast(list[object], result["plan_outcomes"])) == 8 assert result["plan_outcomes_truncated"] is True + + +def test_repair_result_omits_unbounded_receipt_rows_from_outcome_sample() -> None: + outcome = RawReplayPlanOutcome( + "plan-with-receipt", + tuple(f"raw-{index}" for index in range(100)), + RawReplayPlanStatus.EXECUTED, + "done", + "none", + json_document({"application_rows": [{"row": index} for index in range(1000)]}), + ) + result = RepairResult( + "raw_materialization", + MaintenanceCategory.DERIVED_REPAIR, + False, + 1, + True, + plan_outcomes=(outcome,), + ).to_dict() + sample = cast(list[dict[str, object]], result["plan_outcomes"])[0] + assert sample["has_application_receipt"] is True + assert "application_receipt" not in sample + assert sample["input_raw_count"] == 100 + assert len(cast(list[object], sample["input_raw_id_sample"])) == 8 + assert sample["input_raw_id_sample_truncated"] is True + assert "input_raw_ids" not in sample From e10ac19b3a6a6d45d7c968c7266235440f1455a0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 17 Jul 2026 00:19:49 +0200 Subject: [PATCH 08/13] fix(storage): bound raw authority census and ledger reads Advance parser census through a bounded number of authority components per pass. Persist a non-quiescent zero-plan receipt while work remains and publish immutable plans only after the transitive census reaches quiescence. Replace nested census-page payloads with counts, digests, and detail handles. Add chunked canonical-JSON detail reads across CLI and MCP so oversized witnesses, receipts, blockers, and raw-ID lists remain complete without making any response unbounded. Ref polylogue-hjpx.1. --- docs/maintenance.md | 34 ++- .../cli/commands/maintenance/__init__.py | 6 + .../cli/commands/maintenance/_raw_identity.py | 41 +++ polylogue/mcp/server_resources.py | 27 ++ polylogue/storage/raw_authority.py | 262 ++++++++++++++---- polylogue/storage/repair.py | 12 + tests/infra/mcp.py | 1 + .../unit/cli/test_archive_maintenance_cli.py | 72 ++++- tests/unit/mcp/test_server_surfaces.py | 56 +++- .../unit/storage/test_raw_authority_ledger.py | 101 ++++++- tests/unit/storage/test_repair.py | 39 ++- 11 files changed, 582 insertions(+), 69 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index cc1c98f735..d283f01ed0 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -347,13 +347,27 @@ polylogue ops maintenance raw-authority-census \ --output-format json ``` -The response includes both the before-plan inventory and the durable -postflight plan inventory/digests, complete witnesses and outcome for the -current page, and only the blockers created by that census page. -`next_query_handle` advances across the larger inventory without emitting -unbounded blocker history. `--limit` is bounded to 1–500; `--offset` can -override the offset encoded in the URI. MCP clients resolve the URI directly -through the matching resource template. +The response includes bounded before/postflight plan summaries, counts, +digests, and a `detail_query_handle` for each plan. It deliberately does not +inline raw-ID lists, witnesses, preconditions, application receipts, or blocker +documents: one authority component may contain thousands of each. +`next_query_handle` advances across the plan inventory. `--limit` is bounded to +1–500; `--offset` can override the offset encoded in the URI. + +Resolve a census or plan detail handle as bounded canonical-JSON text chunks: + +```bash +polylogue ops maintenance raw-authority-detail \ + 'polylogue://raw-authority-detail/census:42:.../raw-replay:.../0' \ + --chunk-chars 16384 \ + --output-format json +``` + +Concatenate `chunk` values by following `next_query_handle`, then verify the +reconstructed document against `document_sha256`. The chunk size is bounded to +256–65,536 characters. MCP clients resolve both census and detail URIs through +their matching resource templates, so CLI and MCP expose the same complete but +bounded ledger. Every receipt identifies its `mode` (`census`, `dry_run`, or `apply`), whether the parser census was `quiescent`, and its lifecycle. Apply receipts remain @@ -365,6 +379,12 @@ count separately. Finalization also proves that every retryable or carried-forward plan has the identical immutable ID in the postflight census; a partially applied component cannot be mislabeled as unchanged work. +Parser census itself advances through a bounded number of authority components +per pass. If uncensused components remain, the pass persists a non-quiescent +zero-plan census receipt and returns without replay; a later daemon tick resumes +from the per-raw current-parser receipts. Immutable plans are published only +after the complete transitive census is quiescent. + Raw-authority preview is the narrow exception to the generic read-only preview rule above: it may durably record source-tier parser/census observations so a moved-path component has one crash-safe identity across preview and apply. It diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 847167a666..1e81f5dcd6 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -59,6 +59,12 @@ "raw_authority_census_command", "Read a bounded page from a durable raw-authority census ledger.", ), + ( + "raw-authority-detail", + "_raw_identity", + "raw_authority_detail_command", + "Read a bounded chunk of a complete raw-authority ledger record.", + ), ( "raw-authority-blocker-resolve", "_raw_identity", diff --git a/polylogue/cli/commands/maintenance/_raw_identity.py b/polylogue/cli/commands/maintenance/_raw_identity.py index c3e354cbef..703099256a 100644 --- a/polylogue/cli/commands/maintenance/_raw_identity.py +++ b/polylogue/cli/commands/maintenance/_raw_identity.py @@ -65,6 +65,47 @@ def raw_authority_census_command( click.echo(f"Next: {next_handle}") +@click.command("raw-authority-detail") +@click.argument("query_handle") +@click.option("--chunk-chars", type=click.IntRange(256, 65_536), default=16_384, show_default=True) +@click.option("--offset", type=click.IntRange(min=0), default=None) +@click.option( + "--output-format", + "output_format", + type=click.Choice(["plain", "json"]), + default="plain", + show_default=True, +) +@click.pass_obj +def raw_authority_detail_command( + env: AppEnv, + query_handle: str, + chunk_chars: int, + offset: int | None, + output_format: str, +) -> None: + """Read one bounded chunk of a complete census or plan document.""" + del env + from polylogue.storage.raw_authority import read_raw_authority_detail + + try: + payload = read_raw_authority_detail( + archive_root(), + query_handle, + chunk_chars=chunk_chars, + offset=offset, + ) + except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + return + click.echo(str(payload["chunk"]), nl=False) + next_handle = payload.get("next_query_handle") + if next_handle is not None: + click.echo(f"\nNext: {next_handle}") + + @click.command("raw-authority-blocker-resolve") @click.option("--blocker-id", required=True, help="Exact unresolved durable blocker identifier.") @click.option("--reason", required=True, help="Operator rationale recorded in the immutable resolution receipt.") diff --git a/polylogue/mcp/server_resources.py b/polylogue/mcp/server_resources.py index 54a1ea5ef2..182d8c819f 100644 --- a/polylogue/mcp/server_resources.py +++ b/polylogue/mcp/server_resources.py @@ -214,5 +214,32 @@ def raw_authority_census_resource(census_id: str, offset: str) -> str: detail=type(exc).__name__, ) + @mcp.resource("polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}") + def raw_authority_detail_resource(census_id: str, record_id: str, offset: str) -> str: + """Resolve one bounded chunk from a complete census or plan document.""" + try: + from polylogue.storage.raw_authority import read_raw_authority_detail + + root = mcp_archive_root(hooks.get_config()) + handle = f"polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}" + return hooks.json_payload(MCPRootPayload(root=read_raw_authority_detail(root, handle))) + except KeyError: + return hooks.error_json( + f"Raw authority detail not found: {census_id}/{record_id}", + code="not_found", + ) + except (FileNotFoundError, RuntimeError, ValueError) as exc: + return hooks.error_json( + f"Failed to read raw authority detail {census_id}/{record_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + except Exception as exc: + return hooks.error_json( + f"Failed to read raw authority detail {census_id}/{record_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + __all__ = ["register_resources"] diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 5a25dd889a..c662e374b6 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -23,6 +23,8 @@ RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v1" RAW_AUTHORITY_CENSUS_QUERY_PREFIX = "polylogue://raw-authority-census/" +RAW_AUTHORITY_DETAIL_QUERY_PREFIX = "polylogue://raw-authority-detail/" +RAW_AUTHORITY_DETAIL_CHUNK_CHARS = 16_384 logger = get_logger(__name__) @@ -129,6 +131,15 @@ def raw_authority_census_query_handle(census_id: str, *, offset: int = 0) -> str return f"{RAW_AUTHORITY_CENSUS_QUERY_PREFIX}{census_id}/{offset}" +def raw_authority_detail_query_handle(census_id: str, record_id: str, *, offset: int = 0) -> str: + """Return a bounded-chunk URI for one complete census or plan document.""" + if not census_id or "/" in census_id or not record_id or "/" in record_id: + raise ValueError("raw authority detail identifiers must be non-empty and contain no slash") + if offset < 0: + raise ValueError("raw authority detail offset must be non-negative") + return f"{RAW_AUTHORITY_DETAIL_QUERY_PREFIX}{census_id}/{record_id}/{offset}" + + def _raw_authority_census_ref(value: str, *, offset: int | None) -> tuple[str, int]: embedded_offset = 0 if value.startswith(RAW_AUTHORITY_CENSUS_QUERY_PREFIX): @@ -155,6 +166,173 @@ def _decode_json_field(value: object) -> object: return json.loads(value) +def _raw_authority_detail_ref(value: str, *, offset: int | None) -> tuple[str, str, int]: + if not value.startswith(RAW_AUTHORITY_DETAIL_QUERY_PREFIX): + raise ValueError("invalid raw authority detail query handle") + suffix = value.removeprefix(RAW_AUTHORITY_DETAIL_QUERY_PREFIX) + try: + identifiers, encoded_offset = suffix.rsplit("/", 1) + census_id, record_id = identifiers.split("/", 1) + embedded_offset = int(encoded_offset) + except (ValueError, TypeError) as exc: + raise ValueError("invalid raw authority detail query handle") from exc + resolved_offset = embedded_offset if offset is None else offset + if not census_id or "/" in census_id or not record_id or "/" in record_id or resolved_offset < 0: + raise ValueError("invalid raw authority detail query handle") + return census_id, record_id, resolved_offset + + +def _raw_authority_detail_document(conn: sqlite3.Connection, census_id: str, record_id: str) -> JSONDocument: + census = conn.execute( + """ + SELECT census_id, sequence_no, scope_json, residual_json, parser_fingerprint, + mode, lifecycle_status, quiescent, inventory_digest, residual_digest, + plan_count, post_inventory_digest, post_residual_json, + post_residual_digest, post_plan_count, postflight_at_ms, + executable_plan_count, residual_plan_count, predecessor_census_id, + fixed_point, created_at_ms, completed_at_ms + FROM raw_authority_censuses WHERE census_id = ? + """, + (census_id,), + ).fetchone() + if census is None: + raise KeyError(census_id) + if record_id == "census": + return json_document( + { + "record_type": "census", + "census_id": census_id, + "scope": _decode_json_field(census["scope_json"]), + "residual": _decode_json_field(census["residual_json"]), + "post_residual": ( + _decode_json_field(census["post_residual_json"]) + if census["post_residual_json"] is not None + else None + ), + } + ) + row = conn.execute( + """ + SELECT p.plan_id, p.input_digest, p.input_raw_ids_json, + p.logical_keys_json, p.authority_witness_json, + p.source_preconditions_json, p.index_preconditions_json, + p.created_at_ms, cp.ordinal, cp.selected, cp.outcome_status, + cp.reason, cp.next_action, cp.application_receipt_json, + cp.outcome_recorded, cp.recorded_at_ms, + EXISTS( + SELECT 1 FROM raw_authority_census_post_plans AS cpp + WHERE cpp.census_id = ? AND cpp.plan_id = p.plan_id + ) AS present_postflight + FROM raw_authority_plans AS p + LEFT JOIN raw_authority_census_plans AS cp + ON cp.census_id = ? AND cp.plan_id = p.plan_id + WHERE p.plan_id = ? + AND ( + cp.plan_id IS NOT NULL + OR EXISTS( + SELECT 1 FROM raw_authority_census_post_plans AS cpp + WHERE cpp.census_id = ? AND cpp.plan_id = p.plan_id + ) + ) + """, + (census_id, census_id, record_id, census_id), + ).fetchone() + if row is None: + raise KeyError(f"{census_id}/{record_id}") + blockers = [ + { + "blocker_id": str(blocker["blocker_id"]), + "reason": str(blocker["reason"]), + "expected": _decode_json_field(blocker["expected_json"]), + "observed": _decode_json_field(blocker["observed_json"]), + "created_at_ms": int(blocker["created_at_ms"]), + "resolved_at_ms": blocker["resolved_at_ms"], + "resolution": blocker["resolution"], + } + for blocker in conn.execute( + """ + SELECT blocker_id, reason, expected_json, observed_json, + created_at_ms, resolved_at_ms, resolution + FROM raw_authority_blockers + WHERE census_id = ? AND plan_id = ? + ORDER BY created_at_ms, blocker_id + """, + (census_id, record_id), + ) + ] + return json_document( + { + "record_type": "plan", + "census_id": census_id, + "ordinal": row["ordinal"], + "selected": bool(row["selected"]) if row["selected"] is not None else None, + "outcome_status": row["outcome_status"], + "reason": row["reason"], + "next_action": row["next_action"], + "application_receipt": ( + _decode_json_field(row["application_receipt_json"]) + if row["application_receipt_json"] is not None + else None + ), + "outcome_recorded": (bool(row["outcome_recorded"]) if row["outcome_recorded"] is not None else None), + "recorded_at_ms": row["recorded_at_ms"], + "present_postflight": bool(row["present_postflight"]), + "plan": { + "plan_id": str(row["plan_id"]), + "input_digest": str(row["input_digest"]), + "input_raw_ids": _decode_json_field(row["input_raw_ids_json"]), + "logical_keys": _decode_json_field(row["logical_keys_json"]), + "authority_witness": _decode_json_field(row["authority_witness_json"]), + "source_preconditions": _decode_json_field(row["source_preconditions_json"]), + "index_preconditions": _decode_json_field(row["index_preconditions_json"]), + "created_at_ms": int(row["created_at_ms"]), + }, + "blockers": blockers, + } + ) + + +def read_raw_authority_detail( + archive_root: Path, + query_handle: str, + *, + chunk_chars: int = RAW_AUTHORITY_DETAIL_CHUNK_CHARS, + offset: int | None = None, +) -> JSONDocument: + """Read one bounded text chunk of a complete census or plan document.""" + if not 256 <= chunk_chars <= 65_536: + raise ValueError("raw authority detail chunk_chars must be between 256 and 65536") + census_id, record_id, resolved_offset = _raw_authority_detail_ref(query_handle, offset=offset) + source_db = archive_root / "source.db" + if not source_db.is_file(): + raise FileNotFoundError(source_db) + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + conn.row_factory = sqlite3.Row + document = _raw_authority_detail_document(conn, census_id, record_id) + encoded = _canonical_json(document) + if resolved_offset > len(encoded): + raise ValueError("raw authority detail offset exceeds document length") + chunk = encoded[resolved_offset : resolved_offset + chunk_chars] + next_offset = resolved_offset + len(chunk) + return json_document( + { + "query_handle": raw_authority_detail_query_handle(census_id, record_id, offset=resolved_offset), + "next_query_handle": ( + raw_authority_detail_query_handle(census_id, record_id, offset=next_offset) + if next_offset < len(encoded) + else None + ), + "encoding": "canonical-json-text-v1", + "document_sha256": hashlib.sha256(encoded.encode()).hexdigest(), + "document_char_count": len(encoded), + "document_byte_count": len(encoded.encode()), + "offset": resolved_offset, + "chunk_chars": chunk_chars, + "chunk": chunk, + } + ) + + def read_raw_authority_census( archive_root: Path, query_handle: str, @@ -190,12 +368,17 @@ def read_raw_authority_census( rows = conn.execute( """ SELECT cp.ordinal, cp.selected, cp.outcome_status, cp.reason, - cp.next_action, cp.application_receipt_json, cp.recorded_at_ms, + cp.next_action, length(cp.application_receipt_json) AS application_receipt_chars, + cp.recorded_at_ms, cp.outcome_recorded, - p.plan_id, p.input_digest, p.input_raw_ids_json, - p.logical_keys_json, p.authority_witness_json, + p.plan_id, p.input_digest, + json_array_length(p.input_raw_ids_json) AS input_raw_count, + json_array_length(p.logical_keys_json) AS logical_key_count, + p.authority_witness_json, p.source_preconditions_json, p.index_preconditions_json, - p.created_at_ms + p.created_at_ms, + (SELECT COUNT(*) FROM raw_authority_blockers AS b + WHERE b.census_id = cp.census_id AND b.plan_id = cp.plan_id) AS blocker_count FROM raw_authority_census_plans AS cp JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id WHERE cp.census_id = ? @@ -207,7 +390,8 @@ def read_raw_authority_census( post_rows = conn.execute( """ SELECT cpp.ordinal, p.plan_id, p.input_digest, - p.input_raw_ids_json, p.logical_keys_json + json_array_length(p.input_raw_ids_json) AS input_raw_count, + json_array_length(p.logical_keys_json) AS logical_key_count FROM raw_authority_census_post_plans AS cpp JOIN raw_authority_plans AS p ON p.plan_id = cpp.plan_id WHERE cpp.census_id = ? @@ -216,50 +400,28 @@ def read_raw_authority_census( """, (census_id, limit, resolved_offset), ).fetchall() - plan_ids = [str(row["plan_id"]) for row in rows] - blockers: list[dict[str, object]] = [] - if plan_ids: - marks = ",".join("?" for _ in plan_ids) - blockers = [ - { - "blocker_id": str(row["blocker_id"]), - "plan_id": str(row["plan_id"]), - "reason": str(row["reason"]), - "expected": _decode_json_field(row["expected_json"]), - "observed": _decode_json_field(row["observed_json"]), - "created_at_ms": int(row["created_at_ms"]), - "resolved_at_ms": row["resolved_at_ms"], - "resolution": row["resolution"], - } - for row in conn.execute( - f""" - SELECT blocker_id, plan_id, reason, expected_json, - observed_json, created_at_ms, resolved_at_ms, resolution - FROM raw_authority_blockers - WHERE census_id = ? AND plan_id IN ({marks}) - ORDER BY created_at_ms, blocker_id - """, - (census_id, *plan_ids), - ) - ] plans = [ { "ordinal": int(row["ordinal"]), "selected": bool(row["selected"]), "outcome_status": str(row["outcome_status"]), - "reason": str(row["reason"]), - "next_action": str(row["next_action"]), - "application_receipt": _decode_json_field(row["application_receipt_json"]), + "reason_sample": str(row["reason"])[:256], + "reason_chars": len(str(row["reason"])), + "next_action_sample": str(row["next_action"])[:256], + "next_action_chars": len(str(row["next_action"])), + "application_receipt_chars": int(row["application_receipt_chars"]), "outcome_recorded": bool(row["outcome_recorded"]), "recorded_at_ms": int(row["recorded_at_ms"]), + "blocker_count": int(row["blocker_count"]), + "detail_query_handle": raw_authority_detail_query_handle(census_id, str(row["plan_id"])), "plan": { "plan_id": str(row["plan_id"]), "input_digest": str(row["input_digest"]), - "input_raw_ids": _decode_json_field(row["input_raw_ids_json"]), - "logical_keys": _decode_json_field(row["logical_keys_json"]), - "authority_witness": _decode_json_field(row["authority_witness_json"]), - "source_preconditions": _decode_json_field(row["source_preconditions_json"]), - "index_preconditions": _decode_json_field(row["index_preconditions_json"]), + "input_raw_count": int(row["input_raw_count"]), + "logical_key_count": int(row["logical_key_count"]), + "authority_witness_chars": len(str(row["authority_witness_json"])), + "source_preconditions_chars": len(str(row["source_preconditions_json"])), + "index_preconditions_chars": len(str(row["index_preconditions_json"])), "created_at_ms": int(row["created_at_ms"]), }, } @@ -282,8 +444,9 @@ def read_raw_authority_census( "census": { "census_id": str(census["census_id"]), "sequence_no": int(census["sequence_no"]), - "scope": _decode_json_field(census["scope_json"]), - "residual": _decode_json_field(census["residual_json"]), + "detail_query_handle": raw_authority_detail_query_handle(census_id, "census"), + "scope_chars": len(str(census["scope_json"])), + "residual_chars": len(str(census["residual_json"])), "parser_fingerprint": str(census["parser_fingerprint"]), "mode": str(census["mode"]), "lifecycle_status": str(census["lifecycle_status"]), @@ -292,10 +455,8 @@ def read_raw_authority_census( "residual_digest": str(census["residual_digest"]), "plan_count": total, "post_inventory_digest": census["post_inventory_digest"], - "post_residual": ( - _decode_json_field(census["post_residual_json"]) - if census["post_residual_json"] is not None - else None + "post_residual_chars": ( + len(str(census["post_residual_json"])) if census["post_residual_json"] is not None else None ), "post_residual_digest": census["post_residual_digest"], "post_plan_count": post_total, @@ -313,12 +474,13 @@ def read_raw_authority_census( "ordinal": int(row["ordinal"]), "plan_id": str(row["plan_id"]), "input_digest": str(row["input_digest"]), - "input_raw_ids": _decode_json_field(row["input_raw_ids_json"]), - "logical_keys": _decode_json_field(row["logical_keys_json"]), + "input_raw_count": int(row["input_raw_count"]), + "logical_key_count": int(row["logical_key_count"]), + "detail_query_handle": raw_authority_detail_query_handle(census_id, str(row["plan_id"])), } for row in post_rows ], - "blockers": blockers, + "blocker_count": sum(int(row["blocker_count"]) for row in rows), } ) @@ -1272,6 +1434,8 @@ def reject_invalid_raw_replay_application( __all__ = [ "RAW_AUTHORITY_CENSUS_QUERY_PREFIX", + "RAW_AUTHORITY_DETAIL_CHUNK_CHARS", + "RAW_AUTHORITY_DETAIL_QUERY_PREFIX", "RAW_AUTHORITY_PARSER_FINGERPRINT", "RawAuthorityCensusReceipt", "RawReplayPlan", @@ -1282,9 +1446,11 @@ def reject_invalid_raw_replay_application( "finalize_raw_authority_census", "raw_replay_application_receipt", "raw_authority_census_query_handle", + "raw_authority_detail_query_handle", "raw_replay_plan_last_attempts", "recover_interrupted_raw_authority_censuses", "read_raw_authority_census", + "read_raw_authority_detail", "record_raw_authority_census", "record_raw_replay_outcome", "reject_invalid_raw_replay_application", diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 8bf9536133..1b23880db0 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -80,6 +80,7 @@ _PROBE_ONLY_EXACT_MESSAGE_ROW_LIMIT = 100_000 RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES = 1024 * 1024 * 1024 RAW_MATERIALIZATION_RESOURCE_BLOCK_REASON = "non-stream-safe raw payload exceeds the bounded replay limit" +RAW_MATERIALIZATION_CENSUS_COMPONENT_LIMIT = 25 RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT = 8 _TRANSIENT_LOCK_PARSE_ERROR = "OperationalError: database is locked" _QUARANTINED_ACCEPTED_RAW_REPAIR_DETAIL = "repair:accepted_quarantined_raw_exact_byte_and_semantic_proof" @@ -6546,11 +6547,20 @@ def repair_raw_materialization( uncensused_raw_ids = set(uncensused_historical_revision_raw_ids(archive_root, relevant_raw_ids)) census_failed_raw_ids: set[str] = set() census_resource_blocked_raw_ids: set[str] = set() + census_component_limit = ( + raw_artifact_limit if raw_artifact_limit is not None else RAW_MATERIALIZATION_CENSUS_COMPONENT_LIMIT + ) + if census_component_limit < 1: + raise ValueError("raw_artifact_limit must be positive") + census_components_attempted = 0 if uncensused_raw_ids: preliminary_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) for component in preliminary_components: if not uncensused_raw_ids.intersection(component): continue + if census_components_attempted >= census_component_limit: + break + census_components_attempted += 1 seed = _raw_materialization_component_seed(candidates, component) try: census_historical_revision_evidence( @@ -6607,6 +6617,8 @@ def repair_raw_materialization( "raw_materialization_selected_max_blob_bytes": 0.0, "raw_materialization_executed_count": 0.0, "raw_materialization_census_incomplete_raw_count": float(len(census_pending_raw_ids)), + "raw_materialization_census_component_limit": float(census_component_limit), + "raw_materialization_census_components_attempted": float(census_components_attempted), "raw_materialization_census_sequence": float(census_receipt.sequence_no), "raw_materialization_census_fixed_point": 0.0, } diff --git a/tests/infra/mcp.py b/tests/infra/mcp.py index dca0068644..3d82ea6cb9 100644 --- a/tests/infra/mcp.py +++ b/tests/infra/mcp.py @@ -129,6 +129,7 @@ EXPECTED_RESOURCE_TEMPLATE_URIS = { "polylogue://session/{conv_id}", "polylogue://raw-authority-census/{census_id}/{offset}", + "polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}", } EXPECTED_PROMPT_NAMES = { diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index 9f98778dea..3ecf65a687 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -13,11 +13,12 @@ from polylogue.cli.commands.maintenance import _rebuild_index as maintenance_rebuild_index from polylogue.config import Config from polylogue.core.enums import Provider +from polylogue.core.json import json_document from polylogue.maintenance.replay import rebuild_index_from_source from polylogue.sources.live.cursor import CursorStore from polylogue.storage.blob_gc import read_gc_history from polylogue.storage.blob_publication import ArchiveBlobPublisher -from polylogue.storage.raw_authority import record_raw_authority_census +from polylogue.storage.raw_authority import RawReplayPlan, record_raw_authority_census from polylogue.storage.sqlite.archive_tiers.archive import ArchiveSessionSearchHit, ArchiveSessionSummary from polylogue.storage.sqlite.archive_tiers.archive_init import ( ArchiveInitResult, @@ -69,6 +70,75 @@ def test_raw_authority_census_cli_resolves_receipt_handle( assert payload["census"]["census_id"] == receipt.census_id +def test_raw_authority_cli_bounds_oversized_plan_and_resolves_detail( + cli_workspace: dict[str, Path], + cli_runner: CliRunner, +) -> None: + root = cli_workspace["archive_root"] + raw_ids = tuple(f"raw-{index:05d}" for index in range(2_000)) + plan = RawReplayPlan( + "raw-replay:cli-oversized", + "b" * 64, + raw_ids, + ("codex:oversized",), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + ) + receipt = record_raw_authority_census( + root, + (plan,), + selected_plan_ids=set(), + executable_plan_ids={plan.plan_id}, + mode="dry_run", + quiescent=True, + scope={"test": "cli-oversized"}, + residual={}, + ) + + census_result = cli_runner.invoke( + cli, + [ + "--plain", + "ops", + "maintenance", + "raw-authority-census", + receipt.query_handle, + "--limit", + "1", + "--output-format", + "json", + ], + catch_exceptions=False, + ) + assert census_result.exit_code == 0 + assert len(census_result.output) < 8_000 + census_payload = json.loads(census_result.output) + item = census_payload["plans"][0] + assert item["plan"]["input_raw_count"] == 2_000 + assert "raw-01999" not in census_result.output + + detail_result = cli_runner.invoke( + cli, + [ + "--plain", + "ops", + "maintenance", + "raw-authority-detail", + item["detail_query_handle"], + "--chunk-chars", + "256", + "--output-format", + "json", + ], + catch_exceptions=False, + ) + assert detail_result.exit_code == 0 + detail_payload = json.loads(detail_result.output) + assert len(detail_payload["chunk"]) <= 256 + assert detail_payload["next_query_handle"] is not None + + def test_raw_authority_blocker_resolution_cli_requires_confirmation( cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/mcp/test_server_surfaces.py b/tests/unit/mcp/test_server_surfaces.py index 78c965fb99..a648130ff6 100644 --- a/tests/unit/mcp/test_server_surfaces.py +++ b/tests/unit/mcp/test_server_surfaces.py @@ -17,10 +17,11 @@ from polylogue.archive.models import Session, SessionSummary from polylogue.archive.semantic.content_projection import ContentProjectionSpec from polylogue.core.enums import AssertionKind, AssertionStatus, AssertionVisibility, BlockType, BranchType, Provider +from polylogue.core.json import json_document from polylogue.core.refs import EvidenceRef from polylogue.core.types import SessionId from polylogue.sources.parsers.base import ParsedContentBlock, ParsedMessage, ParsedSession -from polylogue.storage.raw_authority import record_raw_authority_census +from polylogue.storage.raw_authority import RawReplayPlan, record_raw_authority_census from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.user_write import ArchiveAssertionEnvelope from polylogue.surfaces.payloads import ( @@ -591,6 +592,59 @@ def test_raw_authority_census_query_handle_resolves_bounded_ledger( assert payload["census"]["census_id"] == receipt.census_id assert payload["plans"] == [] + def test_raw_authority_mcp_bounds_oversized_plan_and_pages_detail( + self: object, mcp_server: MCPServerUnderTest, tmp_path: Path + ) -> None: + archive_root = tmp_path / "archive" + with ArchiveStore(archive_root): + pass + raw_ids = tuple(f"raw-{index:05d}" for index in range(2_000)) + plan = RawReplayPlan( + "raw-replay:mcp-oversized", + "c" * 64, + raw_ids, + ("codex:oversized",), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + ) + receipt = record_raw_authority_census( + archive_root, + (plan,), + selected_plan_ids=set(), + executable_plan_ids={plan.plan_id}, + mode="dry_run", + quiescent=True, + scope={"test": "mcp-oversized"}, + residual={}, + ) + with patch("polylogue.mcp.server._get_config") as mock_get_config: + mock_get_config.return_value = SimpleNamespace( + archive_root=archive_root, + db_path=archive_root / "index.db", + ) + census_result = invoke_surface( + mcp_server._resource_manager._templates["polylogue://raw-authority-census/{census_id}/{offset}"].fn, + census_id=receipt.census_id, + offset="0", + ) + census_payload = json.loads(census_result) + item = census_payload["plans"][0] + assert len(census_result) < 8_000 + assert item["plan"]["input_raw_count"] == 2_000 + detail_result = invoke_surface( + mcp_server._resource_manager._templates[ + "polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}" + ].fn, + census_id=receipt.census_id, + record_id=plan.plan_id, + offset="0", + ) + + detail_payload = json.loads(detail_result) + assert len(detail_payload["chunk"]) <= 16_384 + assert detail_payload["next_query_handle"] is not None + class TestArchiveGenericToolSurfaces: @pytest.mark.asyncio diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 936446171e..bba6f4cc46 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -23,6 +23,7 @@ build_raw_replay_plans, finalize_raw_authority_census, read_raw_authority_census, + read_raw_authority_detail, record_raw_authority_census, record_raw_replay_outcome, reject_stale_raw_replay_plan, @@ -38,6 +39,29 @@ def _config(root: Path) -> Config: return Config(archive_root=root, render_root=root / "render", sources=[], db_path=root / "archive.db") +def _read_detail_document(root: Path, query_handle: str, *, chunk_chars: int = 256) -> dict[str, object]: + chunks: list[str] = [] + handle: str | None = query_handle + digest: str | None = None + for _page in range(10_000): + assert handle is not None + page = read_raw_authority_detail(root, handle, chunk_chars=chunk_chars) + chunk = cast(str, page["chunk"]) + assert len(chunk) <= chunk_chars + chunks.append(chunk) + page_digest = cast(str, page["document_sha256"]) + digest = digest or page_digest + assert page_digest == digest + handle = cast(str | None, page["next_query_handle"]) + if handle is None: + break + else: + raise AssertionError("raw authority detail pagination did not terminate") + document = json.loads("".join(chunks)) + assert isinstance(document, dict) + return cast(dict[str, object], document) + + def _write_codex_raw( root: Path, *, @@ -95,6 +119,11 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat _write_codex_raw(tmp_path, native_id="first", source_path="first.jsonl", acquired_at_ms=1) _write_codex_raw(tmp_path, native_id="second", source_path="second.jsonl", acquired_at_ms=2) + incomplete = repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) + assert incomplete.census_receipt is not None + assert incomplete.census_receipt.quiescent is False + assert incomplete.census_receipt.plan_count == 0 + result = repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) assert result.census_receipt is not None @@ -159,6 +188,11 @@ def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_pat cast(dict[str, object], item)["outcome_status"] for item in (*cast(list[object], first_page["plans"]), *cast(list[object], second_page["plans"])) } == {"executed", "carried_forward"} + first_item = cast(dict[str, object], cast(list[object], first_page["plans"])[0]) + assert "application_receipt" not in first_item + assert "input_raw_ids" not in cast(dict[str, object], first_item["plan"]) + detail = _read_detail_document(tmp_path, cast(str, first_item["detail_query_handle"])) + assert cast(dict[str, object], detail["plan"])["input_raw_ids"] assert cast(dict[str, object], first_page["census"])["post_plan_count"] == 1 assert len(cast(list[object], first_page["post_plans"])) == 1 @@ -322,18 +356,78 @@ def test_global_census_quiesces_moved_component_before_any_plan_is_published(tmp acquired_at_ms=3, ) + incomplete_receipts = [] + for _expected_pass in range(2): + incomplete = repair_raw_materialization(_config(tmp_path), dry_run=True, raw_artifact_limit=1) + assert incomplete.census_receipt is not None + assert incomplete.census_receipt.quiescent is False + assert incomplete.census_receipt.plan_count == 0 + assert incomplete.metrics["raw_materialization_census_component_limit"] == 1.0 + assert incomplete.metrics["raw_materialization_census_components_attempted"] == 1.0 + incomplete_ledger = read_raw_authority_census(tmp_path, incomplete.census_receipt.query_handle) + assert incomplete_ledger["plans"] == [] + incomplete_receipts.append(incomplete.census_receipt.census_id) + assert len(set(incomplete_receipts)) == 2 + preview = repair_raw_materialization(_config(tmp_path), dry_run=True, raw_artifact_limit=1) assert preview.census_receipt is not None assert preview.census_receipt.quiescent is True ledger = read_raw_authority_census(tmp_path, preview.census_receipt.query_handle) raw_sets = { - frozenset(cast(list[str], cast(dict[str, object], cast(dict[str, object], item)["plan"])["input_raw_ids"])) + frozenset( + cast( + list[str], + cast( + dict[str, object], + _read_detail_document( + tmp_path, + cast(str, cast(dict[str, object], item)["detail_query_handle"]), + )["plan"], + )["input_raw_ids"], + ) + ) for item in cast(list[object], ledger["plans"]) } assert raw_sets == {frozenset((first, second)), frozenset((third,))} +def test_census_page_bounds_one_oversized_plan_and_detail_chunks_reconstruct_it(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_ids = tuple(f"raw-{index:05d}" for index in range(2_000)) + plan = RawReplayPlan( + plan_id="raw-replay:oversized", + input_digest="a" * 64, + input_raw_ids=raw_ids, + logical_keys=tuple(f"codex:key-{index:05d}" for index in range(2_000)), + authority_witness=json_document({"rows": [{"raw_id": raw_id} for raw_id in raw_ids]}), + source_preconditions=json_document({"rows": [{"raw_id": raw_id} for raw_id in raw_ids]}), + index_preconditions=json_document({"rows": [{"raw_id": raw_id} for raw_id in raw_ids]}), + ) + receipt = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids=set(), + executable_plan_ids={plan.plan_id}, + mode="dry_run", + quiescent=True, + scope={"test": "oversized"}, + residual={}, + ) + + page = read_raw_authority_census(tmp_path, receipt.query_handle, limit=1) + + assert len(json.dumps(page)) < 8_000 + item = cast(dict[str, object], cast(list[object], page["plans"])[0]) + summary = cast(dict[str, object], item["plan"]) + assert summary["input_raw_count"] == 2_000 + assert "input_raw_ids" not in summary + detail = _read_detail_document(tmp_path, cast(str, item["detail_query_handle"])) + detail_plan = cast(dict[str, object], detail["plan"]) + assert detail_plan["input_raw_ids"] == list(raw_ids) + assert len(cast(list[object], cast(dict[str, object], detail_plan["authority_witness"])["rows"])) == 2_000 + + def test_interrupted_apply_recovers_exact_durable_postconditions(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _write_codex_raw(tmp_path, native_id="crash", source_path="crash.jsonl", acquired_at_ms=1) @@ -510,7 +604,10 @@ def test_identical_stale_rejection_after_resolution_creates_new_open_blocker(tmp conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 ) page = read_raw_authority_census(tmp_path, second_census.query_handle) - blockers = cast(list[object], page["blockers"]) + assert page["blocker_count"] == 1 + item = cast(dict[str, object], cast(list[object], page["plans"])[0]) + detail = _read_detail_document(tmp_path, cast(str, item["detail_query_handle"])) + blockers = cast(list[object], detail["blockers"]) assert len(blockers) == 1 assert cast(dict[str, object], blockers[0])["blocker_id"] == second_blocker diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index c8689f54ba..cdb07582b0 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -24,6 +24,21 @@ def _config(tmp_path: Path) -> Config: return Config(archive_root=tmp_path, render_root=tmp_path, sources=[], db_path=tmp_path / "archive.db") +def _complete_bounded_raw_census(config: Config, *, limit: int) -> tuple[repair_mod.RepairResult, list[str]]: + """Advance census-only passes until a quiescent preview can publish plans.""" + incomplete_census_ids: list[str] = [] + for _pass in range(1_000): + result = repair_mod.repair_raw_materialization(config, dry_run=True, raw_artifact_limit=limit) + assert result.census_receipt is not None + if result.census_receipt.quiescent: + return result, incomplete_census_ids + assert result.census_receipt.plan_count == 0 + attempted = result.metrics["raw_materialization_census_components_attempted"] + assert 1.0 <= attempted <= float(limit) + incomplete_census_ids.append(result.census_receipt.census_id) + raise AssertionError("bounded raw census did not quiesce") + + def _status( *, source_documents: int = 0, @@ -1234,12 +1249,9 @@ def test_raw_materialization_dry_run_reports_limited_selection( conn.commit() config = _config(tmp_path) - result = repair_mod.repair_raw_materialization( - config, - dry_run=True, - raw_artifact_limit=2, - ) + result, incomplete_censuses = _complete_bounded_raw_census(config, limit=2) + assert len(incomplete_censuses) == 1 assert result.success is False assert result.repaired_count == 0 assert "Would: classify and replay" in result.detail @@ -1277,11 +1289,11 @@ def test_raw_materialization_execute_limits_authority_selection( conn.commit() config = _config(tmp_path) - result = repair_mod.repair_raw_materialization( - config, - raw_artifact_limit=2, - ) + preview, incomplete_censuses = _complete_bounded_raw_census(config, limit=2) + result = repair_mod.repair_raw_materialization(config, raw_artifact_limit=2) + assert len(incomplete_censuses) == 1 + assert len(preview.plan_outcomes) == 2 assert result.success is False assert result.repaired_count == 2 assert result.metrics["raw_materialization_candidate_count"] == 4.0 @@ -1887,6 +1899,9 @@ def test_raw_materialization_processes_independent_components_across_bounded_pas assert backlog["execution_blocked"] is False assert backlog["executable_authority_component_count"] == raw_count + preview, incomplete_censuses = _complete_bounded_raw_census(config, limit=5) + assert len(incomplete_censuses) == 4 + assert len(preview.plan_outcomes) == 5 repaired_per_pass: list[int] = [] for _pass in range(5): result = repair_mod.repair_raw_materialization(config, raw_artifact_limit=5) @@ -2027,10 +2042,14 @@ def test_raw_materialization_batch_limit_counts_authority_components(tmp_path: P assert before["candidate_count"] == 9 assert before["authority_component_count"] == 5 - preview = repair_mod.repair_raw_materialization(config, dry_run=True, raw_artifact_limit=3) + preview, incomplete_censuses = _complete_bounded_raw_census(config, limit=3) first = repair_mod.repair_raw_materialization(config, raw_artifact_limit=3) after = repair_mod.raw_materialization_replay_backlog(config) + # The first bounded attempt discovers the five-revision shared component + # transitively; the next pass handles the remaining independent components + # and publishes the complete plan inventory. + assert len(incomplete_censuses) == 1 assert first.repaired_count == 3, (first.detail, first.metrics, after) assert first.metrics["raw_materialization_selected_component_count"] == 3.0 assert first.metrics["raw_materialization_plan_outcome_count"] == 5.0 From 4cf3d9317cec218be802347f6508b1121721cf2a Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 17 Jul 2026 00:23:01 +0200 Subject: [PATCH 09/13] perf(storage): compact bounded census progress receipts Store the pending census backlog as an exact count and stable digest instead of copying every raw ID into each intermediate source-tier receipt. This keeps hundreds of resumable catch-up passes identity-sensitive without quadratic durable growth. Ref polylogue-hjpx.1. --- polylogue/storage/repair.py | 9 ++++++++- tests/unit/storage/test_raw_authority_ledger.py | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index 1b23880db0..938554a19f 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -4975,6 +4975,9 @@ def _raw_authority_residual( resource_blocked_plan_ids: tuple[str, ...] = (), ) -> dict[str, object]: """Return identity-sensitive residual debt for fixed-point comparison.""" + census_pending_digest = hashlib.sha256( + json.dumps(list(census_pending_raw_ids), separators=(",", ":")).encode() + ).hexdigest() return { "missing_blob_raw_ids": list(candidates.missing_blob_raw_ids), "adoption_deferred_raw_ids": list(candidates.adoption_deferred_raw_ids), @@ -4982,7 +4985,11 @@ def _raw_authority_residual( "byte_authority_fragment_raw_ids": list(candidates.byte_authority_fragment_raw_ids), "byte_authority_quarantined_raw_ids": list(candidates.byte_authority_quarantined_raw_ids), "byte_authority_pending_raw_ids": list(candidates.byte_authority_pending_raw_ids), - "census_pending_raw_ids": list(census_pending_raw_ids), + # A large initial catch-up may need hundreds of bounded census passes. + # Keep every progress receipt identity-sensitive without copying the + # entire shrinking raw-ID backlog into source.db on every pass. + "census_pending_raw_count": len(census_pending_raw_ids), + "census_pending_raw_digest": census_pending_digest, "resource_blocked_plan_ids": list(resource_blocked_plan_ids), } diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index bba6f4cc46..04b7b5d2d1 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -366,6 +366,14 @@ def test_global_census_quiesces_moved_component_before_any_plan_is_published(tmp assert incomplete.metrics["raw_materialization_census_components_attempted"] == 1.0 incomplete_ledger = read_raw_authority_census(tmp_path, incomplete.census_receipt.query_handle) assert incomplete_ledger["plans"] == [] + census_detail = _read_detail_document( + tmp_path, + cast(str, cast(dict[str, object], incomplete_ledger["census"])["detail_query_handle"]), + ) + pending_residual = cast(dict[str, object], census_detail["residual"]) + assert pending_residual["census_pending_raw_count"] >= 1 + assert len(cast(str, pending_residual["census_pending_raw_digest"])) == 64 + assert "census_pending_raw_ids" not in pending_residual incomplete_receipts.append(incomplete.census_receipt.census_id) assert len(set(incomplete_receipts)) == 2 From 908a1e41066221adfcac59c63aadd42680c5a996 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 17 Jul 2026 00:24:17 +0200 Subject: [PATCH 10/13] test(storage): type bounded census count assertion Keep the pending-count regression strict under mypy without changing its runtime contract. Ref polylogue-hjpx.1. --- tests/unit/storage/test_raw_authority_ledger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 04b7b5d2d1..02284b4ab0 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -371,7 +371,7 @@ def test_global_census_quiesces_moved_component_before_any_plan_is_published(tmp cast(str, cast(dict[str, object], incomplete_ledger["census"])["detail_query_handle"]), ) pending_residual = cast(dict[str, object], census_detail["residual"]) - assert pending_residual["census_pending_raw_count"] >= 1 + assert cast(int, pending_residual["census_pending_raw_count"]) >= 1 assert len(cast(str, pending_residual["census_pending_raw_digest"])) == 64 assert "census_pending_raw_ids" not in pending_residual incomplete_receipts.append(incomplete.census_receipt.census_id) From ba2283aec42454d7c395ab849a13e983369f05a4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 17 Jul 2026 00:36:34 +0200 Subject: [PATCH 11/13] fix(storage): close raw authority receipt review gaps Problem: final adversarial review found that replay application receipts did not bind accepted application authority to the materialized head, detail pagination could splice changing ledger versions, and blocker resolution returned an unbounded plan payload.\n\nWhat changed: require a current-head application witness, bind every continuation handle to the canonical document digest, fail stale continuations closed, and return bounded blocker summaries while retaining complete resolution evidence behind the detail handle.\n\nVerification: devtools test tests/unit/storage/test_raw_authority_ledger.py; devtools test -k raw_materialization; devtools verify --quick.\n\nRef polylogue-hjpx.1.\n\nCo-Authored-By: Claude --- docs/maintenance.md | 6 +- polylogue/mcp/server_resources.py | 6 +- polylogue/storage/raw_authority.py | 107 +++++++++++++++--- tests/infra/mcp.py | 2 +- tests/unit/mcp/test_server_surfaces.py | 3 +- .../unit/storage/test_raw_authority_ledger.py | 37 ++++++ 6 files changed, 139 insertions(+), 22 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index d283f01ed0..fa024dfe6c 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -358,11 +358,15 @@ Resolve a census or plan detail handle as bounded canonical-JSON text chunks: ```bash polylogue ops maintenance raw-authority-detail \ - 'polylogue://raw-authority-detail/census:42:.../raw-replay:.../0' \ + 'polylogue://raw-authority-detail/census:42:.../raw-replay:.../current/0' \ --chunk-chars 16384 \ --output-format json ``` +The first `current/0` read returns digest-bound continuation handles. If the +underlying outcome or blocker changes between chunks, the old continuation +fails closed; restart from the record's `current/0` handle. + Concatenate `chunk` values by following `next_query_handle`, then verify the reconstructed document against `document_sha256`. The chunk size is bounded to 256–65,536 characters. MCP clients resolve both census and detail URIs through diff --git a/polylogue/mcp/server_resources.py b/polylogue/mcp/server_resources.py index 182d8c819f..77a3537a67 100644 --- a/polylogue/mcp/server_resources.py +++ b/polylogue/mcp/server_resources.py @@ -214,14 +214,14 @@ def raw_authority_census_resource(census_id: str, offset: str) -> str: detail=type(exc).__name__, ) - @mcp.resource("polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}") - def raw_authority_detail_resource(census_id: str, record_id: str, offset: str) -> str: + @mcp.resource("polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}") + def raw_authority_detail_resource(census_id: str, record_id: str, revision: str, offset: str) -> str: """Resolve one bounded chunk from a complete census or plan document.""" try: from polylogue.storage.raw_authority import read_raw_authority_detail root = mcp_archive_root(hooks.get_config()) - handle = f"polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}" + handle = f"polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}" return hooks.json_payload(MCPRootPayload(root=read_raw_authority_detail(root, handle))) except KeyError: return hooks.error_json( diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index c662e374b6..f3ae936835 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -131,13 +131,19 @@ def raw_authority_census_query_handle(census_id: str, *, offset: int = 0) -> str return f"{RAW_AUTHORITY_CENSUS_QUERY_PREFIX}{census_id}/{offset}" -def raw_authority_detail_query_handle(census_id: str, record_id: str, *, offset: int = 0) -> str: +def raw_authority_detail_query_handle( + census_id: str, + record_id: str, + *, + revision: str = "current", + offset: int = 0, +) -> str: """Return a bounded-chunk URI for one complete census or plan document.""" - if not census_id or "/" in census_id or not record_id or "/" in record_id: + if not census_id or "/" in census_id or not record_id or "/" in record_id or not revision or "/" in revision: raise ValueError("raw authority detail identifiers must be non-empty and contain no slash") if offset < 0: raise ValueError("raw authority detail offset must be non-negative") - return f"{RAW_AUTHORITY_DETAIL_QUERY_PREFIX}{census_id}/{record_id}/{offset}" + return f"{RAW_AUTHORITY_DETAIL_QUERY_PREFIX}{census_id}/{record_id}/{revision}/{offset}" def _raw_authority_census_ref(value: str, *, offset: int | None) -> tuple[str, int]: @@ -166,20 +172,30 @@ def _decode_json_field(value: object) -> object: return json.loads(value) -def _raw_authority_detail_ref(value: str, *, offset: int | None) -> tuple[str, str, int]: +def _raw_authority_detail_ref(value: str, *, offset: int | None) -> tuple[str, str, str, int]: if not value.startswith(RAW_AUTHORITY_DETAIL_QUERY_PREFIX): raise ValueError("invalid raw authority detail query handle") suffix = value.removeprefix(RAW_AUTHORITY_DETAIL_QUERY_PREFIX) try: identifiers, encoded_offset = suffix.rsplit("/", 1) - census_id, record_id = identifiers.split("/", 1) + census_id, record_id, revision = identifiers.split("/", 2) embedded_offset = int(encoded_offset) except (ValueError, TypeError) as exc: raise ValueError("invalid raw authority detail query handle") from exc resolved_offset = embedded_offset if offset is None else offset - if not census_id or "/" in census_id or not record_id or "/" in record_id or resolved_offset < 0: + if ( + not census_id + or "/" in census_id + or not record_id + or "/" in record_id + or not revision + or "/" in revision + or resolved_offset < 0 + ): raise ValueError("invalid raw authority detail query handle") - return census_id, record_id, resolved_offset + if revision == "current" and resolved_offset != 0: + raise ValueError("unbound raw authority detail handles may only start at offset zero") + return census_id, record_id, revision, resolved_offset def _raw_authority_detail_document(conn: sqlite3.Connection, census_id: str, record_id: str) -> JSONDocument: @@ -247,7 +263,7 @@ def _raw_authority_detail_document(conn: sqlite3.Connection, census_id: str, rec "observed": _decode_json_field(blocker["observed_json"]), "created_at_ms": int(blocker["created_at_ms"]), "resolved_at_ms": blocker["resolved_at_ms"], - "resolution": blocker["resolution"], + "resolution": (_decode_json_field(blocker["resolution"]) if blocker["resolution"] is not None else None), } for blocker in conn.execute( """ @@ -302,7 +318,7 @@ def read_raw_authority_detail( """Read one bounded text chunk of a complete census or plan document.""" if not 256 <= chunk_chars <= 65_536: raise ValueError("raw authority detail chunk_chars must be between 256 and 65536") - census_id, record_id, resolved_offset = _raw_authority_detail_ref(query_handle, offset=offset) + census_id, record_id, requested_revision, resolved_offset = _raw_authority_detail_ref(query_handle, offset=offset) source_db = archive_root / "source.db" if not source_db.is_file(): raise FileNotFoundError(source_db) @@ -310,20 +326,25 @@ def read_raw_authority_detail( conn.row_factory = sqlite3.Row document = _raw_authority_detail_document(conn, census_id, record_id) encoded = _canonical_json(document) + document_sha256 = hashlib.sha256(encoded.encode()).hexdigest() + if requested_revision != "current" and requested_revision != document_sha256: + raise RuntimeError("raw authority detail changed; restart from its current offset-zero handle") if resolved_offset > len(encoded): raise ValueError("raw authority detail offset exceeds document length") chunk = encoded[resolved_offset : resolved_offset + chunk_chars] next_offset = resolved_offset + len(chunk) return json_document( { - "query_handle": raw_authority_detail_query_handle(census_id, record_id, offset=resolved_offset), + "query_handle": raw_authority_detail_query_handle( + census_id, record_id, revision=document_sha256, offset=resolved_offset + ), "next_query_handle": ( - raw_authority_detail_query_handle(census_id, record_id, offset=next_offset) + raw_authority_detail_query_handle(census_id, record_id, revision=document_sha256, offset=next_offset) if next_offset < len(encoded) else None ), "encoding": "canonical-json-text-v1", - "document_sha256": hashlib.sha256(encoded.encode()).hexdigest(), + "document_sha256": document_sha256, "document_char_count": len(encoded), "document_byte_count": len(encoded.encode()), "offset": resolved_offset, @@ -1000,6 +1021,45 @@ def rows(name: str) -> list[Mapping[str, object]]: problems.append("accepted head content hashes do not match materialized sessions") if any(str(row.get("accepted_raw_id")) not in input_raw_ids for row in head_rows): problems.append("accepted heads do not point into the immutable input component") + heads_by_key = {str(row.get("logical_source_key")): row for row in head_rows} + if len(heads_by_key) != len(head_rows): + problems.append("accepted head receipt contains duplicate logical authority keys") + sessions_by_id: dict[str, Mapping[str, object]] = {} + for session in session_rows: + session_id = str(session.get("session_id")) + previous = sessions_by_id.setdefault(session_id, session) + if previous != session: + problems.append(f"materialized session receipt conflicts for {session_id}") + application_keys: set[str] = set() + applications_matching_current_head: set[str] = set() + for application in application_rows: + key = str(application.get("logical_source_key")) + application_keys.add(key) + head = heads_by_key.get(key) + if head is None: + problems.append(f"application receipt has no accepted head for {key}") + continue + application_authority = ( + str(application.get("session_id")), + str(application.get("accepted_raw_id")), + str(application.get("accepted_content_hash")), + ) + head_authority = ( + str(head.get("session_id")), + str(head.get("accepted_raw_id")), + str(head.get("accepted_content_hash")), + ) + if application_authority == head_authority: + applications_matching_current_head.add(key) + session = sessions_by_id.get(str(head.get("session_id"))) + if session is None: + continue + if str(session.get("raw_id")) != str(head.get("accepted_raw_id")) or str(session.get("content_hash")) != str( + head.get("accepted_content_hash") + ): + problems.append(f"materialized session authority does not match the head for {key}") + for key in sorted(application_keys - applications_matching_current_head): + problems.append(f"no application accepted authority matches the current head for {key}") return not problems, tuple(problems) @@ -1262,7 +1322,7 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu conn.execute("BEGIN IMMEDIATE") row = conn.execute( """ - SELECT b.blocker_id, b.plan_id, b.expected_json, p.input_raw_ids_json + SELECT b.blocker_id, b.plan_id, b.census_id, b.expected_json, p.input_raw_ids_json FROM raw_authority_blockers AS b JOIN raw_authority_plans AS p ON p.plan_id = b.plan_id WHERE b.blocker_id = ? AND b.resolved_at_ms IS NULL @@ -1275,7 +1335,7 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu input_raw_ids = tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))) observed = build_raw_replay_plan(conn, input_raw_ids) now = int(time.time() * 1000) - receipt = json_document( + full_receipt = json_document( { "schema": "polylogue.raw-authority-blocker-resolution.v1", "blocker_id": blocker_id, @@ -1291,13 +1351,28 @@ def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolu SET resolved_at_ms = ?, resolution = ? WHERE blocker_id = ? AND resolved_at_ms IS NULL """, - (now, _canonical_json(receipt), blocker_id), + (now, _canonical_json(full_receipt), blocker_id), ).rowcount if updated != 1: conn.rollback() raise RuntimeError(f"raw authority blocker changed during resolution: {blocker_id}") conn.commit() - return receipt + return json_document( + { + "schema": "polylogue.raw-authority-blocker-resolution-summary.v1", + "blocker_id": blocker_id, + "superseded_plan_id": str(row["plan_id"]), + "current_plan": { + "plan_id": observed.plan_id, + "input_digest": observed.input_digest, + "input_raw_count": len(observed.input_raw_ids), + "logical_key_count": len(observed.logical_keys), + }, + "operator_resolution": resolution.strip(), + "resolved_at_ms": now, + "detail_query_handle": raw_authority_detail_query_handle(str(row["census_id"]), str(row["plan_id"])), + } + ) def reject_stale_raw_replay_plan( diff --git a/tests/infra/mcp.py b/tests/infra/mcp.py index 3d82ea6cb9..258ab5e84f 100644 --- a/tests/infra/mcp.py +++ b/tests/infra/mcp.py @@ -129,7 +129,7 @@ EXPECTED_RESOURCE_TEMPLATE_URIS = { "polylogue://session/{conv_id}", "polylogue://raw-authority-census/{census_id}/{offset}", - "polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}", + "polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}", } EXPECTED_PROMPT_NAMES = { diff --git a/tests/unit/mcp/test_server_surfaces.py b/tests/unit/mcp/test_server_surfaces.py index a648130ff6..58be4a5519 100644 --- a/tests/unit/mcp/test_server_surfaces.py +++ b/tests/unit/mcp/test_server_surfaces.py @@ -634,10 +634,11 @@ def test_raw_authority_mcp_bounds_oversized_plan_and_pages_detail( assert item["plan"]["input_raw_count"] == 2_000 detail_result = invoke_surface( mcp_server._resource_manager._templates[ - "polylogue://raw-authority-detail/{census_id}/{record_id}/{offset}" + "polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}" ].fn, census_id=receipt.census_id, record_id=plan.plan_id, + revision="current", offset="0", ) diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py index 02284b4ab0..5ceb3aaa75 100644 --- a/tests/unit/storage/test_raw_authority_ledger.py +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -490,6 +490,23 @@ def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: ) +@pytest.mark.parametrize("field", ["session_id", "accepted_raw_id", "accepted_content_hash"]) +def test_application_receipt_requires_exact_application_authority(tmp_path: Path, field: str) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id=f"exact-{field}", source_path=f"{field}.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).success is True + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + receipt = dict(raw_authority_mod.raw_replay_application_receipt(tmp_path, plan)) + application_rows = cast(list[dict[str, object]], receipt["application_rows"]) + assert application_rows + application_rows[0][field] = f"wrong-{field}" + + valid, problems = raw_authority_mod.validate_raw_replay_application_receipt(plan, receipt) + + assert valid is False + assert any("no application accepted authority matches" in problem for problem in problems) + + def test_recovery_rejects_partial_expanded_membership_postconditions(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _write_codex_raw( @@ -556,10 +573,30 @@ def test_stale_blocker_resolution_replans_current_evidence_and_resumes(tmp_path: rejected = reject_stale_raw_replay_plan(tmp_path, census.census_id, plan, observed) blocker_id = cast(str, cast(dict[str, object], rejected.application_receipt)["blocker_id"]) + census_page = read_raw_authority_census(tmp_path, census.query_handle) + plan_summary = cast(dict[str, object], cast(list[object], census_page["plans"])[0]) + first_detail_page = read_raw_authority_detail( + tmp_path, + cast(str, plan_summary["detail_query_handle"]), + chunk_chars=256, + ) + stale_continuation = cast(str, first_detail_page["next_query_handle"]) + resolution = resolve_raw_authority_blocker(tmp_path, blocker_id, resolution="current path is authoritative") + with pytest.raises(RuntimeError, match="raw authority detail changed"): + read_raw_authority_detail(tmp_path, stale_continuation, chunk_chars=256) + current_detail = _read_detail_document(tmp_path, cast(str, resolution["detail_query_handle"])) resumed = repair_raw_materialization(_config(tmp_path)) assert resolution["blocker_id"] == blocker_id + resolution_plan = cast(dict[str, object], resolution["current_plan"]) + assert resolution_plan["input_raw_count"] == 1 + assert "input_raw_ids" not in resolution_plan + stored_resolution = cast( + dict[str, object], + cast(dict[str, object], cast(list[object], current_detail["blockers"])[0])["resolution"], + ) + assert cast(dict[str, object], stored_resolution["current_plan"])["input_raw_ids"] == [raw_id] assert resumed.success is True assert resumed.repaired_count == 1 assert "raw_materialization_unresolved_blocker_count" not in resumed.metrics From 07120f83ceb3231cccac0943314da3510ae3e413 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 17 Jul 2026 00:37:51 +0200 Subject: [PATCH 12/13] fix(storage): disambiguate session receipt validation Keep the exact receipt validation typed under the repository's strict mypy gate.\n\nVerification: mypy polylogue/storage/raw_authority.py.\n\nRef polylogue-hjpx.1.\n\nCo-Authored-By: Claude --- polylogue/storage/raw_authority.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index f3ae936835..a1d10a05fe 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -1027,8 +1027,8 @@ def rows(name: str) -> list[Mapping[str, object]]: sessions_by_id: dict[str, Mapping[str, object]] = {} for session in session_rows: session_id = str(session.get("session_id")) - previous = sessions_by_id.setdefault(session_id, session) - if previous != session: + existing_session = sessions_by_id.setdefault(session_id, session) + if existing_session != session: problems.append(f"materialized session receipt conflicts for {session_id}") application_keys: set[str] = set() applications_matching_current_head: set[str] = set() From 34ed698553bc88e1d0ed6a6e9c9dc7da31c09971 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 17 Jul 2026 00:38:43 +0200 Subject: [PATCH 13/13] fix(storage): avoid receipt validator type shadowing Use a distinct name for the optional materialized-session lookup so strict mypy preserves its narrowing.\n\nVerification: mypy polylogue/storage/raw_authority.py.\n\nRef polylogue-hjpx.1.\n\nCo-Authored-By: Claude --- polylogue/storage/raw_authority.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index a1d10a05fe..e2d771bca2 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -1051,12 +1051,12 @@ def rows(name: str) -> list[Mapping[str, object]]: ) if application_authority == head_authority: applications_matching_current_head.add(key) - session = sessions_by_id.get(str(head.get("session_id"))) - if session is None: + materialized_session = sessions_by_id.get(str(head.get("session_id"))) + if materialized_session is None: continue - if str(session.get("raw_id")) != str(head.get("accepted_raw_id")) or str(session.get("content_hash")) != str( - head.get("accepted_content_hash") - ): + if str(materialized_session.get("raw_id")) != str(head.get("accepted_raw_id")) or str( + materialized_session.get("content_hash") + ) != str(head.get("accepted_content_hash")): problems.append(f"materialized session authority does not match the head for {key}") for key in sorted(application_keys - applications_matching_current_head): problems.append(f"no application accepted authority matches the current head for {key}")