diff --git a/docs/maintenance.md b/docs/maintenance.md index ca5787cf4a..fa024dfe6c 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -334,6 +334,81 @@ structural (missing columns, corrupted index file, or a broken write path). Stop the daemon, restore from backup or rebuild the affected index tier, and open an issue with the probe output attached. +### Inspecting a raw-authority census + +Raw source-to-index convergence records an immutable census in `source.db`. +Status and daemon receipts expose a bounded summary plus a URI such as +`polylogue://raw-authority-census/census:42:.../0`. Resolve that same URI from +the CLI without copying every plan into the status payload: + +```bash +polylogue ops maintenance raw-authority-census \ + 'polylogue://raw-authority-census/census:42:.../0' \ + --output-format json +``` + +The response includes bounded before/postflight plan summaries, counts, +digests, and a `detail_query_handle` for each plan. It deliberately does not +inline raw-ID lists, witnesses, preconditions, application receipts, or blocker +documents: one authority component may contain thousands of each. +`next_query_handle` advances across the plan inventory. `--limit` is bounded to +1–500; `--offset` can override the offset encoded in the URI. + +Resolve a census or plan detail handle as bounded canonical-JSON text chunks: + +```bash +polylogue ops maintenance raw-authority-detail \ + 'polylogue://raw-authority-detail/census:42:.../raw-replay:.../current/0' \ + --chunk-chars 16384 \ + --output-format json +``` + +The first `current/0` read returns digest-bound continuation handles. If the +underlying outcome or blocker changes between chunks, the old continuation +fails closed; restart from the record's `current/0` handle. + +Concatenate `chunk` values by following `next_query_handle`, then verify the +reconstructed document against `document_sha256`. The chunk size is bounded to +256–65,536 characters. MCP clients resolve both census and detail URIs through +their matching resource templates, so CLI and MCP expose the same complete but +bounded ledger. + +Every receipt identifies its `mode` (`census`, `dry_run`, or `apply`), whether +the parser census was `quiescent`, and its lifecycle. Apply receipts remain +`planned` until every selected immutable plan has an outcome; startup recovery +then validates exact source, application/membership, accepted-head, and session +postconditions before marking an interrupted pass `executed`. Readiness never +reports a `planned` row as the latest completed census and exposes its pending +count separately. Finalization also proves that every retryable or +carried-forward plan has the identical immutable ID in the postflight census; +a partially applied component cannot be mislabeled as unchanged work. + +Parser census itself advances through a bounded number of authority components +per pass. If uncensused components remain, the pass persists a non-quiescent +zero-plan census receipt and returns without replay; a later daemon tick resumes +from the per-raw current-parser receipts. Immutable plans are published only +after the complete transitive census is quiescent. + +Raw-authority preview is the narrow exception to the generic read-only preview +rule above: it may durably record source-tier parser/census observations so a +moved-path component has one crash-safe identity across preview and apply. It +never selects or applies an index replay plan. + +A stale precondition or incomplete application receipt creates a durable, +fail-closed blocker. After inspecting the census URI and current evidence, +explicitly reopen replanning with a recorded rationale: + +```bash +polylogue ops maintenance raw-authority-blocker-resolve \ + --blocker-id 'raw-authority-blocker:...' \ + --reason 'reviewed current source/index evidence; replan from this state' \ + --yes +``` + +Resolution never applies the stale plan. It stores the replacement plan +witness in the resolution receipt; the next ordinary convergence pass plans +and validates current evidence normally. + ### Draining the convergence-debt queue **Symptoms.** `polylogue ops diagnostics workload` reports a non-trivial diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index 693abe2ccc..d7af104f60 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -586,7 +586,7 @@ files: owner: archive-session reason: archive-domain semantics - path: polylogue/archive/session/extraction.py - loc: 571 + loc: 445 target: polylogue/archive/session/extraction.py owner: archive-session reason: archive-domain semantics @@ -611,7 +611,7 @@ files: owner: archive-session reason: archive-domain semantics - path: polylogue/archive/session/runtime.py - loc: 584 + loc: 587 target: polylogue/archive/session/runtime.py owner: archive-session reason: archive-domain semantics @@ -720,7 +720,7 @@ files: target: polylogue/browser_capture/actions.py owner: stable - path: polylogue/browser_capture/capture_jobs.py - loc: 366 + loc: 594 target: polylogue/browser_capture/capture_jobs.py owner: stable - path: polylogue/browser_capture/identity.py @@ -736,11 +736,11 @@ files: target: polylogue/browser_capture/receiver.py owner: stable - path: polylogue/browser_capture/route_contracts.py - loc: 203 + loc: 279 target: polylogue/browser_capture/route_contracts.py owner: stable - path: polylogue/browser_capture/server.py - loc: 693 + loc: 723 target: polylogue/browser_capture/server.py owner: stable - path: polylogue/cli/__init__.py @@ -940,7 +940,7 @@ files: target: polylogue/cli/commands/scan_secrets.py owner: stable - path: polylogue/cli/commands/status.py - loc: 2522 + loc: 2551 target: polylogue/cli/commands/status.py owner: stable - path: polylogue/cli/commands/status_diagnostics.py @@ -1413,7 +1413,7 @@ files: target: polylogue/daemon/catchup_status.py owner: stable - path: polylogue/daemon/cli.py - loc: 1956 + loc: 1974 target: polylogue/daemon/cli.py owner: stable - path: polylogue/daemon/compare.py @@ -1871,7 +1871,7 @@ files: target: polylogue/insights/resume.py owner: stable - path: polylogue/insights/rigor.py - loc: 1091 + loc: 1067 target: polylogue/insights/rigor.py owner: stable - path: polylogue/insights/run_projection.py @@ -3162,7 +3162,7 @@ files: owner: stable cross_cut: { lifecycle: model } - path: polylogue/sources/revision_backfill.py - loc: 337 + loc: 410 target: polylogue/sources/revision_backfill.py owner: stable - path: polylogue/sources/source_acquisition.py @@ -3203,7 +3203,7 @@ files: target: TBD owner: storage-domain - path: polylogue/storage/archive_readiness.py - loc: 686 + loc: 722 target: TBD owner: storage-domain - path: polylogue/storage/archive_views.py @@ -3290,7 +3290,7 @@ files: target: polylogue/storage/embeddings/embedding_stats.py owner: stable - path: polylogue/storage/embeddings/materialization.py - loc: 1327 + loc: 1330 target: polylogue/storage/embeddings/materialization.py owner: stable - path: polylogue/storage/embeddings/models.py @@ -3409,7 +3409,7 @@ files: target: polylogue/storage/insights/session/records.py owner: stable - path: polylogue/storage/insights/session/refresh.py - loc: 613 + loc: 688 target: polylogue/storage/insights/session/refresh.py owner: stable - path: polylogue/storage/insights/session/repair_assessment.py @@ -3483,13 +3483,17 @@ files: loc: 137 target: polylogue/storage/raw/models.py owner: stable + - path: polylogue/storage/raw_authority.py + loc: 547 + target: TBD + owner: storage-domain - path: polylogue/storage/raw_retention.py loc: 1317 target: polylogue/storage/raw_retention.py owner: storage-root reason: storage-root cross-cutting helper - path: polylogue/storage/repair.py - loc: 6909 + loc: 7015 target: polylogue/storage/repair.py owner: storage-root reason: storage-root cross-cutting helper @@ -3697,7 +3701,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/__init__.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive.py - loc: 11407 + loc: 11412 target: polylogue/storage/sqlite/archive_tiers/archive.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive_init.py @@ -3769,7 +3773,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/session_annotations_write.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/source.py - loc: 323 + loc: 396 target: polylogue/storage/sqlite/archive_tiers/source.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/source_write.py diff --git a/docs/topology-status.md b/docs/topology-status.md index 093c61e55d..946e4d6d93 100644 --- a/docs/topology-status.md +++ b/docs/topology-status.md @@ -31,9 +31,9 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe - **Stable** (no move scoped): 834 - **Kernel** (polylogue/ root): 7 - **Primitives** (storage-root): 18 -- **TBD** (cell needs explicit assignment): 7 -- **Total declared**: 995 -- **Realized polylogue/**/*.py**: 995 files declared +- **TBD** (cell needs explicit assignment): 8 +- **Total declared**: 996 +- **Realized polylogue/**/*.py**: 996 files declared ### TBD cells (require explicit routing) @@ -45,5 +45,6 @@ These rows in the projection have no resolved target yet. Each needs an explicit - `polylogue/storage/blob_publication.py` — no rule yet - `polylogue/storage/block_anchor.py` — no rule yet - `polylogue/storage/index_generation.py` — no rule yet +- `polylogue/storage/raw_authority.py` — no rule yet - `polylogue/storage/table_existence.py` — no rule yet diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 921683bdcf..1e81f5dcd6 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -53,6 +53,24 @@ "rebuild_index_command", "Inspect or execute an authority-safe source-to-index rebuild.", ), + ( + "raw-authority-census", + "_raw_identity", + "raw_authority_census_command", + "Read a bounded page from a durable raw-authority census ledger.", + ), + ( + "raw-authority-detail", + "_raw_identity", + "raw_authority_detail_command", + "Read a bounded chunk of a complete raw-authority ledger record.", + ), + ( + "raw-authority-blocker-resolve", + "_raw_identity", + "raw_authority_blocker_resolve_command", + "Resolve one stale-plan blocker against current source evidence.", + ), ( "missing-raw-blob-cursors", "_raw_identity", diff --git a/polylogue/cli/commands/maintenance/_raw_identity.py b/polylogue/cli/commands/maintenance/_raw_identity.py index 60481ad48e..703099256a 100644 --- a/polylogue/cli/commands/maintenance/_raw_identity.py +++ b/polylogue/cli/commands/maintenance/_raw_identity.py @@ -17,6 +17,133 @@ from polylogue.paths import archive_root, render_root +@click.command("raw-authority-census") +@click.argument("query_handle") +@click.option("--limit", type=click.IntRange(1, 500), default=100, show_default=True) +@click.option("--offset", type=click.IntRange(min=0), default=None) +@click.option( + "--output-format", + "output_format", + type=click.Choice(["plain", "json"]), + default="plain", + show_default=True, +) +@click.pass_obj +def raw_authority_census_command( + env: AppEnv, + query_handle: str, + limit: int, + offset: int | None, + output_format: str, +) -> None: + """Read a bounded page from a durable raw-authority census ledger.""" + del env + from polylogue.storage.raw_authority import read_raw_authority_census + + try: + payload = read_raw_authority_census(archive_root(), query_handle, limit=limit, offset=offset) + except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + return + census = payload["census"] + if not isinstance(census, dict): + raise click.ClickException("invalid raw authority census payload") + click.echo( + f"Census {census['census_id']}: plans={census['plan_count']} " + f"executable={census['executable_plan_count']} residual={census['residual_plan_count']} " + f"fixed_point={str(census['fixed_point']).lower()}" + ) + for item in payload["plans"] if isinstance(payload["plans"], list) else []: + if isinstance(item, dict): + plan = item.get("plan") + plan_id = plan.get("plan_id") if isinstance(plan, dict) else "unknown" + click.echo(f" {item.get('ordinal')} {item.get('outcome_status')} {plan_id}") + next_handle = payload.get("next_query_handle") + if next_handle is not None: + click.echo(f"Next: {next_handle}") + + +@click.command("raw-authority-detail") +@click.argument("query_handle") +@click.option("--chunk-chars", type=click.IntRange(256, 65_536), default=16_384, show_default=True) +@click.option("--offset", type=click.IntRange(min=0), default=None) +@click.option( + "--output-format", + "output_format", + type=click.Choice(["plain", "json"]), + default="plain", + show_default=True, +) +@click.pass_obj +def raw_authority_detail_command( + env: AppEnv, + query_handle: str, + chunk_chars: int, + offset: int | None, + output_format: str, +) -> None: + """Read one bounded chunk of a complete census or plan document.""" + del env + from polylogue.storage.raw_authority import read_raw_authority_detail + + try: + payload = read_raw_authority_detail( + archive_root(), + query_handle, + chunk_chars=chunk_chars, + offset=offset, + ) + except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + return + click.echo(str(payload["chunk"]), nl=False) + next_handle = payload.get("next_query_handle") + if next_handle is not None: + click.echo(f"\nNext: {next_handle}") + + +@click.command("raw-authority-blocker-resolve") +@click.option("--blocker-id", required=True, help="Exact unresolved durable blocker identifier.") +@click.option("--reason", required=True, help="Operator rationale recorded in the immutable resolution receipt.") +@click.option("--yes", "confirmed", is_flag=True, help="Confirm resolving this blocker against current evidence.") +@click.option( + "--output-format", + "output_format", + type=click.Choice(["plain", "json"]), + default="plain", + show_default=True, +) +@click.pass_obj +def raw_authority_blocker_resolve_command( + env: AppEnv, + blocker_id: str, + reason: str, + confirmed: bool, + output_format: str, +) -> None: + """Resolve one stale-plan blocker after replanning current evidence.""" + del env + if not confirmed: + raise click.ClickException("refusing to resolve a durable blocker without --yes") + from polylogue.storage.raw_authority import resolve_raw_authority_blocker + + try: + receipt = resolve_raw_authority_blocker(archive_root(), blocker_id, resolution=reason) + except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(receipt, indent=2, sort_keys=True)) + return + click.echo(f"Resolved {blocker_id}") + current_plan = receipt.get("current_plan") + if isinstance(current_plan, dict): + click.echo(f"Current plan: {current_plan.get('plan_id', 'unknown')}") + + def _raw_blob_path_for_hash(root: Path, blob_hash: bytes | str) -> Path | None: hex_hash = blob_hash.hex() if isinstance(blob_hash, bytes) else str(blob_hash).lower() if len(hex_hash) != 64 or any(char not in "0123456789abcdef" for char in hex_hash): diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index 49dc71195f..93f1554b5b 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -639,19 +639,46 @@ def _drain_raw_materialization_once(*, limit: int = _RAW_MATERIALIZATION_CONVERG def _emit_raw_materialization_pass(result: Any) -> None: """Persist the conserved plan outcomes for one bounded daemon pass.""" outcomes = tuple(getattr(result, "plan_outcomes", ())) + outcome_sample_limit = 8 from polylogue.daemon.events import emit_daemon_event metrics = dict(getattr(result, "metrics", {})) + census = getattr(result, "census_receipt", None) + census_payload = None + if census is not None: + census_payload = { + "census_id": census.census_id, + "sequence_no": census.sequence_no, + "inventory_digest": census.inventory_digest, + "residual_digest": census.residual_digest, + "plan_count": census.plan_count, + "post_inventory_digest": census.post_inventory_digest, + "post_residual_digest": census.post_residual_digest, + "post_plan_count": census.post_plan_count, + "executable_plan_count": census.executable_plan_count, + "residual_plan_count": census.residual_plan_count, + "predecessor_census_id": census.predecessor_census_id, + "mode": census.mode, + "lifecycle_status": census.lifecycle_status, + "quiescent": census.quiescent, + "fixed_point": census.fixed_point, + "query_handle": census.query_handle, + } + payload = { + "pass_id": f"raw-materialization:{os.urandom(16).hex()}", + "success": bool(result.success), + "repaired_count": int(result.repaired_count), + "detail": str(result.detail), + "metrics": metrics, + "plan_outcome_count": len(outcomes), + "plan_outcome_sample": [outcome.to_summary_dict() for outcome in outcomes[:outcome_sample_limit]], + "plan_outcome_sample_truncated": len(outcomes) > outcome_sample_limit, + } + if census_payload is not None: + payload["census"] = census_payload emit_daemon_event( "raw_materialization_pass", - payload={ - "pass_id": f"raw-materialization:{os.urandom(16).hex()}", - "success": bool(result.success), - "repaired_count": int(result.repaired_count), - "detail": str(result.detail), - "metrics": metrics, - "plan_outcomes": [outcome.to_dict() for outcome in outcomes], - }, + payload=payload, ) diff --git a/polylogue/mcp/server_resources.py b/polylogue/mcp/server_resources.py index 1805974a2a..77a3537a67 100644 --- a/polylogue/mcp/server_resources.py +++ b/polylogue/mcp/server_resources.py @@ -190,5 +190,56 @@ def readiness_resource() -> str: exclude_none=True, ) + @mcp.resource("polylogue://raw-authority-census/{census_id}/{offset}") + def raw_authority_census_resource(census_id: str, offset: str) -> str: + """Resolve one bounded page from a durable raw-authority ledger.""" + try: + from polylogue.storage.raw_authority import read_raw_authority_census + + root = mcp_archive_root(hooks.get_config()) + handle = f"polylogue://raw-authority-census/{census_id}/{offset}" + return hooks.json_payload(MCPRootPayload(root=read_raw_authority_census(root, handle))) + except KeyError: + return hooks.error_json(f"Raw authority census not found: {census_id}", code="not_found") + except (FileNotFoundError, RuntimeError, ValueError) as exc: + return hooks.error_json( + f"Failed to read raw authority census {census_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + except Exception as exc: + return hooks.error_json( + f"Failed to read raw authority census {census_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + + @mcp.resource("polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}") + def raw_authority_detail_resource(census_id: str, record_id: str, revision: str, offset: str) -> str: + """Resolve one bounded chunk from a complete census or plan document.""" + try: + from polylogue.storage.raw_authority import read_raw_authority_detail + + root = mcp_archive_root(hooks.get_config()) + handle = f"polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}" + return hooks.json_payload(MCPRootPayload(root=read_raw_authority_detail(root, handle))) + except KeyError: + return hooks.error_json( + f"Raw authority detail not found: {census_id}/{record_id}", + code="not_found", + ) + except (FileNotFoundError, RuntimeError, ValueError) as exc: + return hooks.error_json( + f"Failed to read raw authority detail {census_id}/{record_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + except Exception as exc: + return hooks.error_json( + f"Failed to read raw authority detail {census_id}/{record_id}: {exc}", + code="internal_error", + detail=type(exc).__name__, + ) + __all__ = ["register_resources"] diff --git a/polylogue/sources/revision_backfill.py b/polylogue/sources/revision_backfill.py index 2b445e30de..c557b7f4a9 100644 --- a/polylogue/sources/revision_backfill.py +++ b/polylogue/sources/revision_backfill.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import pickle import sqlite3 @@ -36,6 +37,25 @@ class RevisionBackfillResult: adoption_deferred: int = 0 +@dataclass(frozen=True, slots=True) +class RevisionCensusResult: + scanned: int + classified_full: int + quarantined: int + input_raw_ids: tuple[str, ...] + logical_keys: tuple[str, ...] + + +@dataclass(slots=True) +class _RevisionCensusState: + scanned: int + classified: int + quarantined: int + censused: set[str] + membership_candidates: dict[str, set[str]] + provisional_full_raw_ids: dict[str, set[str]] + + class RawRevisionReplayResourceBlockedError(RuntimeError): def __init__(self, raw_ids: list[str], limit_bytes: int, total_bytes: int) -> None: self.raw_ids = tuple(raw_ids) @@ -44,6 +64,217 @@ def __init__(self, raw_ids: list[str], limit_bytes: int, total_bytes: int) -> No super().__init__(f"{len(raw_ids)} raw revision(s) total {total_bytes} bytes exceed replay limit {limit_bytes}") +def uncensused_historical_revision_raw_ids(archive_root: Path, raw_ids: list[str]) -> tuple[str, ...]: + """Return inputs whose current parser identity has not been persisted. + + The dedicated receipt proves that the current parser actually observed + every relevant raw. Durable revision or membership rows alone may have + been produced by an older parser and therefore cannot establish current + quiescence. + """ + if not raw_ids: + return () + placeholders = ",".join("?" for _ in raw_ids) + with sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True) as conn: + rows = conn.execute( + f""" + SELECT r.raw_id + FROM raw_sessions AS r + LEFT JOIN raw_authority_parser_census AS c ON c.raw_id = r.raw_id + WHERE r.raw_id IN ({placeholders}) + AND NOT COALESCE( + c.parser_fingerprint = 'revision-membership-v1' + AND c.status = 'complete', + 0 + ) + ORDER BY r.raw_id + """, + raw_ids, + ).fetchall() + return tuple(str(row[0]) for row in rows) + + +def _record_raw_authority_parser_census(archive_root: Path, raw_ids: tuple[str, ...]) -> None: + """Persist per-raw current-parser completion without changing governance.""" + if not raw_ids: + return + with sqlite3.connect(archive_root / "source.db") as conn, conn: + for raw_id in raw_ids: + raw = conn.execute( + """ + SELECT logical_source_key, revision_kind + FROM raw_sessions WHERE raw_id = ? + """, + (raw_id,), + ).fetchone() + membership_census = conn.execute( + """ + SELECT status, detail FROM raw_membership_census + WHERE raw_id = ? AND parser_fingerprint = 'revision-membership-v1' + """, + (raw_id,), + ).fetchone() + membership_keys = [ + str(row[0]) + for row in conn.execute( + """ + SELECT logical_source_key FROM raw_session_memberships + WHERE raw_id = ? ORDER BY logical_source_key + """, + (raw_id,), + ) + ] + typed_key = ( + str(raw[0]) + if raw is not None and raw[0] is not None and str(raw[1]) != RawRevisionKind.UNKNOWN.value + else None + ) + complete = typed_key is not None or ( + membership_census is not None and str(membership_census[0]) in {"complete", "non_session"} + ) + logical_keys = sorted(set(membership_keys) | ({typed_key} if typed_key is not None else set())) + detail = ( + "current parser established durable authority identity" + if complete + else ( + str(membership_census[1]) + if membership_census is not None + else "current parser produced no durable authority identity" + ) + ) + conn.execute( + """ + INSERT INTO raw_authority_parser_census ( + raw_id, parser_fingerprint, status, logical_keys_json, + detail, censused_at_ms + ) VALUES (?, 'revision-membership-v1', ?, ?, ?, 0) + ON CONFLICT(raw_id) DO UPDATE SET + parser_fingerprint = excluded.parser_fingerprint, + status = excluded.status, + logical_keys_json = excluded.logical_keys_json, + detail = excluded.detail, + censused_at_ms = excluded.censused_at_ms + """, + (raw_id, "complete" if complete else "failed", json.dumps(logical_keys), detail), + ) + + +def _census_historical_revision_evidence( + archive: ArchiveStore, + spill: _ParsedSessionSpill, + *, + selected_raw_ids: list[str] | None, + max_payload_bytes: int | None, +) -> _RevisionCensusState: + """Persist a complete bounded parser census without mutating index.db.""" + state = _RevisionCensusState(0, 0, 0, set(), {}, {}) + census_selections: tuple[tuple[str, ...] | None, ...] + if selected_raw_ids is None: + census_selections = (None,) + else: + census_selections = archive.raw_membership_selection_components(selected_raw_ids) + for initial_selection in census_selections: + census_selection = initial_selection + while True: + rows = archive.raw_membership_census_rows(census_selection) + pending_rows = [(raw_id, source_index) for raw_id, source_index in rows if raw_id not in state.censused] + if max_payload_bytes is not None: + payload_sizes = archive.raw_payload_sizes([raw_id for raw_id, _index in rows]) + total_payload_bytes = sum(payload_sizes.values()) + oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] + if oversized or total_payload_bytes > max_payload_bytes: + blocked_ids = oversized or list(payload_sizes) + raise RawRevisionReplayResourceBlockedError( + sorted(blocked_ids), max_payload_bytes, total_payload_bytes + ) + for raw_id, source_index in pending_rows: + state.scanned += 1 + state.censused.add(raw_id) + if source_index < 0: + archive.replace_raw_membership_census( + raw_id, + None, + parser_fingerprint="revision-membership-v1", + censused_at_ms=0, + detail=BYTE_AUTHORITY_CENSUS_DETAIL, + ) + state.quarantined += 1 + continue + try: + sessions, payload_bytes, revision_kind = _parse_retained_raw(archive, raw_id) + except Exception as exc: + archive.replace_raw_membership_census( + raw_id, + None, + parser_fingerprint="revision-membership-v1", + censused_at_ms=0, + detail=str(exc), + ) + state.quarantined += 1 + continue + state.classified += int(len(sessions) == 1) + spill.add(raw_id, sessions, payload_bytes=payload_bytes) + if len(sessions) == 1 and revision_kind is RawRevisionKind.UNKNOWN: + session = sessions[0] + logical_key = f"{session.source_name.value}:{session.provider_session_id}" + archive.bind_raw_revision( + raw_id, + RawRevisionEnvelope( + logical_source_key=logical_key, + kind=RawRevisionKind.FULL, + source_revision=raw_id, + acquisition_generation=0, + authority=RawRevisionAuthority.QUARANTINED, + ), + ) + state.provisional_full_raw_ids.setdefault(logical_key, set()).add(raw_id) + elif revision_kind is RawRevisionKind.UNKNOWN: + archive.replace_raw_membership_census( + raw_id, + sessions, + parser_fingerprint="revision-membership-v1", + censused_at_ms=0, + ) + for session in sessions: + logical_key = f"{session.source_name.value}:{session.provider_session_id}" + state.membership_candidates.setdefault(logical_key, set()).add(raw_id) + if census_selection is None: + break + expanded, _keys = archive.expand_raw_membership_selection(list(census_selection)) + if set(expanded) == set(census_selection): + break + census_selection = expanded + return state + + +def census_historical_revision_evidence( + archive_root: Path, + *, + selected_raw_ids: list[str] | None = None, + max_payload_bytes: int | None = None, +) -> RevisionCensusResult: + """Complete the source-tier census stage without applying index changes.""" + with ( + ArchiveStore.open_existing(archive_root, read_only=False) as archive, + _ParsedSessionSpill(archive_root) as spill, + ): + state = _census_historical_revision_evidence( + archive, + spill, + selected_raw_ids=selected_raw_ids, + max_payload_bytes=max_payload_bytes, + ) + expanded, logical_keys = archive.expand_raw_membership_selection(selected_raw_ids) + _record_raw_authority_parser_census(archive_root, tuple(expanded)) + return RevisionCensusResult( + state.scanned, + state.classified, + state.quarantined, + expanded, + logical_keys, + ) + + def backfill_historical_revision_evidence( archive_root: Path, *, @@ -58,13 +289,9 @@ def backfill_historical_revision_evidence( loaded one logical authority cohort at a time. Peak retained session trees therefore follow the largest raw/cohort, not the archive-wide raw count. """ - scanned = 0 - classified = 0 - quarantined = 0 adoption_deferred = 0 + quarantined = 0 logical_keys: set[str] = set() - membership_candidates: dict[str, set[str]] = {} - provisional_full_raw_ids: dict[str, set[str]] = {} archive_context = ( ArchiveStore.open_owned_inactive_generation( archive_root, @@ -75,83 +302,21 @@ def backfill_historical_revision_evidence( else ArchiveStore.open_existing(archive_root, read_only=False) ) with archive_context as archive, _ParsedSessionSpill(archive_root) as spill: - census_selections: tuple[tuple[str, ...] | None, ...] - if selected_raw_ids is None: - census_selections = (None,) - else: - census_selections = archive.raw_membership_selection_components(selected_raw_ids) - censused: set[str] = set() - for initial_selection in census_selections: - census_selection = initial_selection - while True: - rows = archive.raw_membership_census_rows(census_selection) - pending_rows = [(raw_id, source_index) for raw_id, source_index in rows if raw_id not in censused] - if max_payload_bytes is not None: - payload_sizes = archive.raw_payload_sizes([raw_id for raw_id, _index in rows]) - total_payload_bytes = sum(payload_sizes.values()) - oversized = [raw_id for raw_id, size in payload_sizes.items() if size > max_payload_bytes] - if oversized or total_payload_bytes > max_payload_bytes: - blocked_ids = oversized or list(payload_sizes) - raise RawRevisionReplayResourceBlockedError( - sorted(blocked_ids), max_payload_bytes, total_payload_bytes - ) - for raw_id, source_index in pending_rows: - scanned += 1 - censused.add(raw_id) - if source_index < 0: - archive.replace_raw_membership_census( - raw_id, - None, - parser_fingerprint="revision-membership-v1", - censused_at_ms=0, - detail=BYTE_AUTHORITY_CENSUS_DETAIL, - ) - quarantined += 1 - continue - try: - sessions, _payload_bytes, revision_kind = _parse_retained_raw(archive, raw_id) - except Exception as exc: - archive.replace_raw_membership_census( - raw_id, - None, - parser_fingerprint="revision-membership-v1", - censused_at_ms=0, - detail=str(exc), - ) - quarantined += 1 - continue - classified += int(len(sessions) == 1) - spill.add(raw_id, sessions, payload_bytes=_payload_bytes) - if len(sessions) == 1 and revision_kind is RawRevisionKind.UNKNOWN: - session = sessions[0] - logical_key = f"{session.source_name.value}:{session.provider_session_id}" - archive.bind_raw_revision( - raw_id, - RawRevisionEnvelope( - logical_source_key=logical_key, - kind=RawRevisionKind.FULL, - source_revision=raw_id, - acquisition_generation=0, - authority=RawRevisionAuthority.QUARANTINED, - ), - ) - provisional_full_raw_ids.setdefault(logical_key, set()).add(raw_id) - elif revision_kind is RawRevisionKind.UNKNOWN: - archive.replace_raw_membership_census( - raw_id, - sessions, - parser_fingerprint="revision-membership-v1", - censused_at_ms=0, - ) - for session in sessions: - logical_key = f"{session.source_name.value}:{session.provider_session_id}" - membership_candidates.setdefault(logical_key, set()).add(raw_id) - if census_selection is None: - break - expanded, _keys = archive.expand_raw_membership_selection(list(census_selection)) - if set(expanded) == set(census_selection): - break - census_selection = expanded + census = _census_historical_revision_evidence( + archive, + spill, + selected_raw_ids=selected_raw_ids, + max_payload_bytes=max_payload_bytes, + ) + censused_raw_ids, _censused_keys = archive.expand_raw_membership_selection(selected_raw_ids) + # The direct backfill entry point must publish the same current-parser + # receipt as the census-only entry point before it assigns or applies + # any index plan. Commit the source census first so the separate + # durable receipt writer observes one complete source snapshot. + archive.commit() + _record_raw_authority_parser_census(archive_root, tuple(censused_raw_ids)) + membership_candidates = census.membership_candidates + provisional_full_raw_ids = census.provisional_full_raw_ids _unclassified, selected_keys = archive.raw_revision_rebuild_selection(selected_raw_ids) logical_keys.update(selected_keys) @@ -245,7 +410,13 @@ def backfill_historical_revision_evidence( ) if classification.accepted_raw_ids: replayed += 1 - return RevisionBackfillResult(scanned, classified, replayed, quarantined, adoption_deferred) + return RevisionBackfillResult( + census.scanned, + census.classified, + replayed, + census.quarantined + quarantined, + adoption_deferred, + ) def _parse_retained_raw(archive: ArchiveStore, raw_id: str) -> tuple[list[ParsedSession], int, RawRevisionKind]: @@ -334,4 +505,10 @@ def _parse_one(provider: Provider, payload: bytes, source_path: str) -> list[Par ) -__all__ = ["RawRevisionReplayResourceBlockedError", "RevisionBackfillResult", "backfill_historical_revision_evidence"] +__all__ = [ + "RawRevisionReplayResourceBlockedError", + "RevisionBackfillResult", + "RevisionCensusResult", + "backfill_historical_revision_evidence", + "census_historical_revision_evidence", +] diff --git a/polylogue/storage/archive_readiness.py b/polylogue/storage/archive_readiness.py index 9a730fd59d..5391cccc58 100644 --- a/polylogue/storage/archive_readiness.py +++ b/polylogue/storage/archive_readiness.py @@ -228,6 +228,57 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob ) lost_source_evidence_count = _missing_source_raw_session_count(conn) lost_source_evidence_samples = _missing_source_raw_session_samples(conn) + authority_census: dict[str, object] | None = None + authority_pending_census_count = 0 + if _table_columns(conn, "source", "raw_authority_censuses"): + authority_pending_census_count = int( + conn.execute( + """ + SELECT COUNT(*) FROM source.raw_authority_censuses + WHERE lifecycle_status = 'planned' + """ + ).fetchone()[0] + ) + census_row = conn.execute( + """ + SELECT census_id, sequence_no, inventory_digest, residual_digest, + plan_count, post_inventory_digest, post_residual_digest, + post_plan_count, executable_plan_count, residual_plan_count, + predecessor_census_id, mode, lifecycle_status, quiescent, + fixed_point, completed_at_ms + FROM source.raw_authority_censuses + WHERE lifecycle_status IN ('completed', 'interrupted') + ORDER BY sequence_no DESC LIMIT 1 + """ + ).fetchone() + if census_row is not None: + authority_census = { + "census_id": str(census_row["census_id"]), + "sequence_no": int(census_row["sequence_no"]), + "inventory_digest": str(census_row["inventory_digest"]), + "residual_digest": str(census_row["residual_digest"]), + "plan_count": int(census_row["plan_count"]), + "post_inventory_digest": str(census_row["post_inventory_digest"]), + "post_residual_digest": str(census_row["post_residual_digest"]), + "post_plan_count": int(census_row["post_plan_count"]), + "executable_plan_count": int(census_row["executable_plan_count"]), + "residual_plan_count": int(census_row["residual_plan_count"]), + "predecessor_census_id": census_row["predecessor_census_id"], + "mode": str(census_row["mode"]), + "lifecycle_status": str(census_row["lifecycle_status"]), + "quiescent": bool(census_row["quiescent"]), + "fixed_point": bool(census_row["fixed_point"]), + "completed_at_ms": int(census_row["completed_at_ms"]), + "pending_census_count": authority_pending_census_count, + "query_handle": (f"polylogue://raw-authority-census/{census_row['census_id']}/0"), + } + authority_blocker_count = 0 + if _table_columns(conn, "source", "raw_authority_blockers"): + authority_blocker_count = int( + conn.execute( + "SELECT COUNT(*) FROM source.raw_authority_blockers WHERE resolved_at_ms IS NULL" + ).fetchone()[0] + ) except Exception as exc: return { "available": False, @@ -286,6 +337,13 @@ def raw_materialization_readiness_snapshot(active_archive: Path) -> dict[str, ob "lost_source_evidence_samples": lost_source_evidence_samples, "category_counts": category_counts, "source_family_counts": {str(item["origin"]): int(item["count"] or 0) for item in family_rows}, + "raw_authority_census": authority_census, + "raw_authority_blocker_count": authority_blocker_count, + "raw_authority_pending_census_count": authority_pending_census_count, + "raw_authority_ledger_counts": { + "unresolved_blockers": authority_blocker_count, + "pending_censuses": authority_pending_census_count, + }, } diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py new file mode 100644 index 0000000000..e2d771bca2 --- /dev/null +++ b/polylogue/storage/raw_authority.py @@ -0,0 +1,1537 @@ +"""Durable immutable plans and conservation receipts for raw reconciliation. + +The source tier is the authority for this ledger. ``index.db`` may be rebuilt +and ``ops.db`` may be deleted; neither event is allowed to erase replay +fairness, stale-plan blockers, or the proof that a complete census was +accounted for. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import time +from collections.abc import Mapping, Sequence +from contextlib import closing +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path + +from polylogue.core.json import JSONDocument, json_document +from polylogue.logging import get_logger + +RAW_AUTHORITY_PARSER_FINGERPRINT = "revision-membership-v1" +RAW_AUTHORITY_CENSUS_QUERY_PREFIX = "polylogue://raw-authority-census/" +RAW_AUTHORITY_DETAIL_QUERY_PREFIX = "polylogue://raw-authority-detail/" +RAW_AUTHORITY_DETAIL_CHUNK_CHARS = 16_384 +logger = get_logger(__name__) + + +class RawReplayPlanStatus(StrEnum): + EXECUTED = "executed" + RETRYABLE = "retryable" + DEFERRED = "deferred" + TERMINAL = "terminal" + REJECTED_STALE = "rejected_stale" + CARRIED_FORWARD = "carried_forward" + + +@dataclass(frozen=True, slots=True) +class RawReplayPlan: + plan_id: str + input_digest: str + input_raw_ids: tuple[str, ...] + logical_keys: tuple[str, ...] + authority_witness: JSONDocument + source_preconditions: JSONDocument + index_preconditions: JSONDocument + + def to_dict(self) -> JSONDocument: + return json_document( + { + "plan_id": self.plan_id, + "input_digest": self.input_digest, + "input_raw_ids": list(self.input_raw_ids), + "logical_keys": list(self.logical_keys), + "authority_witness": self.authority_witness, + "source_preconditions": self.source_preconditions, + "index_preconditions": self.index_preconditions, + } + ) + + +@dataclass(frozen=True, slots=True) +class RawReplayPlanOutcome: + plan_id: str + input_raw_ids: tuple[str, ...] + status: RawReplayPlanStatus + reason: str + next_action: str + application_receipt: JSONDocument | None = None + + def to_dict(self) -> JSONDocument: + payload: dict[str, object] = { + "plan_id": self.plan_id, + "input_raw_ids": list(self.input_raw_ids), + "status": self.status.value, + "reason": self.reason, + "next_action": self.next_action, + } + if self.application_receipt is not None: + payload["application_receipt"] = self.application_receipt + return json_document(payload) + + def to_summary_dict(self) -> JSONDocument: + """Return a bounded projection; the census handle owns full receipts.""" + raw_id_sample_limit = 8 + return json_document( + { + "plan_id": self.plan_id, + "input_raw_count": len(self.input_raw_ids), + "input_raw_id_sample": list(self.input_raw_ids[:raw_id_sample_limit]), + "input_raw_id_sample_truncated": len(self.input_raw_ids) > raw_id_sample_limit, + "status": self.status.value, + "reason": self.reason, + "next_action": self.next_action, + "has_application_receipt": self.application_receipt is not None, + } + ) + + +@dataclass(frozen=True, slots=True) +class RawAuthorityCensusReceipt: + census_id: str + sequence_no: int + inventory_digest: str + residual_digest: str + plan_count: int + executable_plan_count: int + residual_plan_count: int + post_inventory_digest: str | None + post_residual_digest: str | None + post_plan_count: int | None + predecessor_census_id: str | None + mode: str + lifecycle_status: str + quiescent: bool + fixed_point: bool + + @property + def query_handle(self) -> str: + return raw_authority_census_query_handle(self.census_id) + + +def raw_authority_census_query_handle(census_id: str, *, offset: int = 0) -> str: + """Return a directly resolvable, paginated census-ledger URI.""" + if not census_id or "/" in census_id: + raise ValueError("raw authority census id must be non-empty and contain no slash") + if offset < 0: + raise ValueError("raw authority census offset must be non-negative") + return f"{RAW_AUTHORITY_CENSUS_QUERY_PREFIX}{census_id}/{offset}" + + +def raw_authority_detail_query_handle( + census_id: str, + record_id: str, + *, + revision: str = "current", + offset: int = 0, +) -> str: + """Return a bounded-chunk URI for one complete census or plan document.""" + if not census_id or "/" in census_id or not record_id or "/" in record_id or not revision or "/" in revision: + raise ValueError("raw authority detail identifiers must be non-empty and contain no slash") + if offset < 0: + raise ValueError("raw authority detail offset must be non-negative") + return f"{RAW_AUTHORITY_DETAIL_QUERY_PREFIX}{census_id}/{record_id}/{revision}/{offset}" + + +def _raw_authority_census_ref(value: str, *, offset: int | None) -> tuple[str, int]: + embedded_offset = 0 + if value.startswith(RAW_AUTHORITY_CENSUS_QUERY_PREFIX): + suffix = value.removeprefix(RAW_AUTHORITY_CENSUS_QUERY_PREFIX) + try: + census_id, encoded_offset = suffix.rsplit("/", 1) + embedded_offset = int(encoded_offset) + except (ValueError, TypeError) as exc: + raise ValueError("invalid raw authority census query handle") from exc + elif value.startswith("raw-authority-census:"): + # Read compatibility for the short-lived pre-URI receipt shape. + census_id = value.removeprefix("raw-authority-census:") + else: + census_id = value + resolved_offset = embedded_offset if offset is None else offset + if not census_id or "/" in census_id or resolved_offset < 0: + raise ValueError("invalid raw authority census query handle") + return census_id, resolved_offset + + +def _decode_json_field(value: object) -> object: + if not isinstance(value, str): + raise RuntimeError("raw authority ledger contains a non-text JSON field") + return json.loads(value) + + +def _raw_authority_detail_ref(value: str, *, offset: int | None) -> tuple[str, str, str, int]: + if not value.startswith(RAW_AUTHORITY_DETAIL_QUERY_PREFIX): + raise ValueError("invalid raw authority detail query handle") + suffix = value.removeprefix(RAW_AUTHORITY_DETAIL_QUERY_PREFIX) + try: + identifiers, encoded_offset = suffix.rsplit("/", 1) + census_id, record_id, revision = identifiers.split("/", 2) + embedded_offset = int(encoded_offset) + except (ValueError, TypeError) as exc: + raise ValueError("invalid raw authority detail query handle") from exc + resolved_offset = embedded_offset if offset is None else offset + if ( + not census_id + or "/" in census_id + or not record_id + or "/" in record_id + or not revision + or "/" in revision + or resolved_offset < 0 + ): + raise ValueError("invalid raw authority detail query handle") + if revision == "current" and resolved_offset != 0: + raise ValueError("unbound raw authority detail handles may only start at offset zero") + return census_id, record_id, revision, resolved_offset + + +def _raw_authority_detail_document(conn: sqlite3.Connection, census_id: str, record_id: str) -> JSONDocument: + census = conn.execute( + """ + SELECT census_id, sequence_no, scope_json, residual_json, parser_fingerprint, + mode, lifecycle_status, quiescent, inventory_digest, residual_digest, + plan_count, post_inventory_digest, post_residual_json, + post_residual_digest, post_plan_count, postflight_at_ms, + executable_plan_count, residual_plan_count, predecessor_census_id, + fixed_point, created_at_ms, completed_at_ms + FROM raw_authority_censuses WHERE census_id = ? + """, + (census_id,), + ).fetchone() + if census is None: + raise KeyError(census_id) + if record_id == "census": + return json_document( + { + "record_type": "census", + "census_id": census_id, + "scope": _decode_json_field(census["scope_json"]), + "residual": _decode_json_field(census["residual_json"]), + "post_residual": ( + _decode_json_field(census["post_residual_json"]) + if census["post_residual_json"] is not None + else None + ), + } + ) + row = conn.execute( + """ + SELECT p.plan_id, p.input_digest, p.input_raw_ids_json, + p.logical_keys_json, p.authority_witness_json, + p.source_preconditions_json, p.index_preconditions_json, + p.created_at_ms, cp.ordinal, cp.selected, cp.outcome_status, + cp.reason, cp.next_action, cp.application_receipt_json, + cp.outcome_recorded, cp.recorded_at_ms, + EXISTS( + SELECT 1 FROM raw_authority_census_post_plans AS cpp + WHERE cpp.census_id = ? AND cpp.plan_id = p.plan_id + ) AS present_postflight + FROM raw_authority_plans AS p + LEFT JOIN raw_authority_census_plans AS cp + ON cp.census_id = ? AND cp.plan_id = p.plan_id + WHERE p.plan_id = ? + AND ( + cp.plan_id IS NOT NULL + OR EXISTS( + SELECT 1 FROM raw_authority_census_post_plans AS cpp + WHERE cpp.census_id = ? AND cpp.plan_id = p.plan_id + ) + ) + """, + (census_id, census_id, record_id, census_id), + ).fetchone() + if row is None: + raise KeyError(f"{census_id}/{record_id}") + blockers = [ + { + "blocker_id": str(blocker["blocker_id"]), + "reason": str(blocker["reason"]), + "expected": _decode_json_field(blocker["expected_json"]), + "observed": _decode_json_field(blocker["observed_json"]), + "created_at_ms": int(blocker["created_at_ms"]), + "resolved_at_ms": blocker["resolved_at_ms"], + "resolution": (_decode_json_field(blocker["resolution"]) if blocker["resolution"] is not None else None), + } + for blocker in conn.execute( + """ + SELECT blocker_id, reason, expected_json, observed_json, + created_at_ms, resolved_at_ms, resolution + FROM raw_authority_blockers + WHERE census_id = ? AND plan_id = ? + ORDER BY created_at_ms, blocker_id + """, + (census_id, record_id), + ) + ] + return json_document( + { + "record_type": "plan", + "census_id": census_id, + "ordinal": row["ordinal"], + "selected": bool(row["selected"]) if row["selected"] is not None else None, + "outcome_status": row["outcome_status"], + "reason": row["reason"], + "next_action": row["next_action"], + "application_receipt": ( + _decode_json_field(row["application_receipt_json"]) + if row["application_receipt_json"] is not None + else None + ), + "outcome_recorded": (bool(row["outcome_recorded"]) if row["outcome_recorded"] is not None else None), + "recorded_at_ms": row["recorded_at_ms"], + "present_postflight": bool(row["present_postflight"]), + "plan": { + "plan_id": str(row["plan_id"]), + "input_digest": str(row["input_digest"]), + "input_raw_ids": _decode_json_field(row["input_raw_ids_json"]), + "logical_keys": _decode_json_field(row["logical_keys_json"]), + "authority_witness": _decode_json_field(row["authority_witness_json"]), + "source_preconditions": _decode_json_field(row["source_preconditions_json"]), + "index_preconditions": _decode_json_field(row["index_preconditions_json"]), + "created_at_ms": int(row["created_at_ms"]), + }, + "blockers": blockers, + } + ) + + +def read_raw_authority_detail( + archive_root: Path, + query_handle: str, + *, + chunk_chars: int = RAW_AUTHORITY_DETAIL_CHUNK_CHARS, + offset: int | None = None, +) -> JSONDocument: + """Read one bounded text chunk of a complete census or plan document.""" + if not 256 <= chunk_chars <= 65_536: + raise ValueError("raw authority detail chunk_chars must be between 256 and 65536") + census_id, record_id, requested_revision, resolved_offset = _raw_authority_detail_ref(query_handle, offset=offset) + source_db = archive_root / "source.db" + if not source_db.is_file(): + raise FileNotFoundError(source_db) + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + conn.row_factory = sqlite3.Row + document = _raw_authority_detail_document(conn, census_id, record_id) + encoded = _canonical_json(document) + document_sha256 = hashlib.sha256(encoded.encode()).hexdigest() + if requested_revision != "current" and requested_revision != document_sha256: + raise RuntimeError("raw authority detail changed; restart from its current offset-zero handle") + if resolved_offset > len(encoded): + raise ValueError("raw authority detail offset exceeds document length") + chunk = encoded[resolved_offset : resolved_offset + chunk_chars] + next_offset = resolved_offset + len(chunk) + return json_document( + { + "query_handle": raw_authority_detail_query_handle( + census_id, record_id, revision=document_sha256, offset=resolved_offset + ), + "next_query_handle": ( + raw_authority_detail_query_handle(census_id, record_id, revision=document_sha256, offset=next_offset) + if next_offset < len(encoded) + else None + ), + "encoding": "canonical-json-text-v1", + "document_sha256": document_sha256, + "document_char_count": len(encoded), + "document_byte_count": len(encoded.encode()), + "offset": resolved_offset, + "chunk_chars": chunk_chars, + "chunk": chunk, + } + ) + + +def read_raw_authority_census( + archive_root: Path, + query_handle: str, + *, + limit: int = 100, + offset: int | None = None, +) -> JSONDocument: + """Read one bounded page from an immutable source-tier census ledger.""" + if not 1 <= limit <= 500: + raise ValueError("raw authority census limit must be between 1 and 500") + census_id, resolved_offset = _raw_authority_census_ref(query_handle, offset=offset) + source_db = archive_root / "source.db" + if not source_db.is_file(): + raise FileNotFoundError(source_db) + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + conn.row_factory = sqlite3.Row + census = conn.execute( + """ + SELECT census_id, sequence_no, scope_json, residual_json, parser_fingerprint, + mode, lifecycle_status, quiescent, + inventory_digest, residual_digest, plan_count, + post_inventory_digest, post_residual_json, + post_residual_digest, post_plan_count, postflight_at_ms, + executable_plan_count, residual_plan_count, + predecessor_census_id, fixed_point, created_at_ms, + completed_at_ms + FROM raw_authority_censuses WHERE census_id = ? + """, + (census_id,), + ).fetchone() + if census is None: + raise KeyError(census_id) + rows = conn.execute( + """ + SELECT cp.ordinal, cp.selected, cp.outcome_status, cp.reason, + cp.next_action, length(cp.application_receipt_json) AS application_receipt_chars, + cp.recorded_at_ms, + cp.outcome_recorded, + p.plan_id, p.input_digest, + json_array_length(p.input_raw_ids_json) AS input_raw_count, + json_array_length(p.logical_keys_json) AS logical_key_count, + p.authority_witness_json, + p.source_preconditions_json, p.index_preconditions_json, + p.created_at_ms, + (SELECT COUNT(*) FROM raw_authority_blockers AS b + WHERE b.census_id = cp.census_id AND b.plan_id = cp.plan_id) AS blocker_count + FROM raw_authority_census_plans AS cp + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE cp.census_id = ? + ORDER BY cp.ordinal + LIMIT ? OFFSET ? + """, + (census_id, limit, resolved_offset), + ).fetchall() + post_rows = conn.execute( + """ + SELECT cpp.ordinal, p.plan_id, p.input_digest, + json_array_length(p.input_raw_ids_json) AS input_raw_count, + json_array_length(p.logical_keys_json) AS logical_key_count + FROM raw_authority_census_post_plans AS cpp + JOIN raw_authority_plans AS p ON p.plan_id = cpp.plan_id + WHERE cpp.census_id = ? + ORDER BY cpp.ordinal + LIMIT ? OFFSET ? + """, + (census_id, limit, resolved_offset), + ).fetchall() + plans = [ + { + "ordinal": int(row["ordinal"]), + "selected": bool(row["selected"]), + "outcome_status": str(row["outcome_status"]), + "reason_sample": str(row["reason"])[:256], + "reason_chars": len(str(row["reason"])), + "next_action_sample": str(row["next_action"])[:256], + "next_action_chars": len(str(row["next_action"])), + "application_receipt_chars": int(row["application_receipt_chars"]), + "outcome_recorded": bool(row["outcome_recorded"]), + "recorded_at_ms": int(row["recorded_at_ms"]), + "blocker_count": int(row["blocker_count"]), + "detail_query_handle": raw_authority_detail_query_handle(census_id, str(row["plan_id"])), + "plan": { + "plan_id": str(row["plan_id"]), + "input_digest": str(row["input_digest"]), + "input_raw_count": int(row["input_raw_count"]), + "logical_key_count": int(row["logical_key_count"]), + "authority_witness_chars": len(str(row["authority_witness_json"])), + "source_preconditions_chars": len(str(row["source_preconditions_json"])), + "index_preconditions_chars": len(str(row["index_preconditions_json"])), + "created_at_ms": int(row["created_at_ms"]), + }, + } + for row in rows + ] + total = int(census["plan_count"]) + post_total = int(census["post_plan_count"] or 0) + next_offset = resolved_offset + max(len(plans), len(post_rows)) + return json_document( + { + "query_handle": raw_authority_census_query_handle(census_id, offset=resolved_offset), + "next_query_handle": ( + raw_authority_census_query_handle(census_id, offset=next_offset) + if next_offset < max(total, post_total) + else None + ), + "offset": resolved_offset, + "limit": limit, + "returned_count": len(plans), + "census": { + "census_id": str(census["census_id"]), + "sequence_no": int(census["sequence_no"]), + "detail_query_handle": raw_authority_detail_query_handle(census_id, "census"), + "scope_chars": len(str(census["scope_json"])), + "residual_chars": len(str(census["residual_json"])), + "parser_fingerprint": str(census["parser_fingerprint"]), + "mode": str(census["mode"]), + "lifecycle_status": str(census["lifecycle_status"]), + "quiescent": bool(census["quiescent"]), + "inventory_digest": str(census["inventory_digest"]), + "residual_digest": str(census["residual_digest"]), + "plan_count": total, + "post_inventory_digest": census["post_inventory_digest"], + "post_residual_chars": ( + len(str(census["post_residual_json"])) if census["post_residual_json"] is not None else None + ), + "post_residual_digest": census["post_residual_digest"], + "post_plan_count": post_total, + "postflight_at_ms": census["postflight_at_ms"], + "executable_plan_count": int(census["executable_plan_count"]), + "residual_plan_count": int(census["residual_plan_count"]), + "predecessor_census_id": census["predecessor_census_id"], + "fixed_point": bool(census["fixed_point"]), + "created_at_ms": int(census["created_at_ms"]), + "completed_at_ms": (int(census["completed_at_ms"]) if census["completed_at_ms"] is not None else None), + }, + "plans": plans, + "post_plans": [ + { + "ordinal": int(row["ordinal"]), + "plan_id": str(row["plan_id"]), + "input_digest": str(row["input_digest"]), + "input_raw_count": int(row["input_raw_count"]), + "logical_key_count": int(row["logical_key_count"]), + "detail_query_handle": raw_authority_detail_query_handle(census_id, str(row["plan_id"])), + } + for row in post_rows + ], + "blocker_count": sum(int(row["blocker_count"]) for row in rows), + } + ) + + +def _canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +def _json_value(value: object) -> object: + if isinstance(value, bytes): + return value.hex() + return value + + +def _rows(conn: sqlite3.Connection, sql: str, params: Sequence[object] = ()) -> list[dict[str, object]]: + cursor = conn.execute(sql, tuple(params)) + names = tuple(column[0] for column in cursor.description or ()) + return [{name: _json_value(value) for name, value in zip(names, row, strict=True)} for row in cursor] + + +def build_raw_replay_plan(conn: sqlite3.Connection, input_raw_ids: Sequence[str]) -> RawReplayPlan: + """Snapshot one complete component from an attached source/index pair.""" + raw_ids = tuple(sorted(dict.fromkeys(input_raw_ids))) + if not raw_ids: + raise ValueError("raw replay plan requires at least one input raw id") + marks = ",".join("?" for _ in raw_ids) + source_rows = _rows( + conn, + f""" + SELECT raw_id, origin, native_id, source_path, source_index, + hex(blob_hash) AS blob_hash, blob_size, logical_source_key, + revision_kind, source_revision, predecessor_source_revision, + predecessor_raw_id, baseline_raw_id, append_start_offset, + append_end_offset, acquisition_generation, revision_authority + FROM raw_sessions WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + raw_ids, + ) + if tuple(str(row["raw_id"]) for row in source_rows) != raw_ids: + raise RuntimeError("raw replay plan input disappeared during census") + membership_rows = _rows( + conn, + f""" + SELECT raw_id, logical_source_key, provider_session_id, source_revision, + hex(normalized_content_hash) AS normalized_content_hash, + message_count, predecessor_raw_id, acquisition_generation, + revision_authority, decision + FROM raw_session_memberships + WHERE raw_id IN ({marks}) + ORDER BY raw_id, logical_source_key + """, + raw_ids, + ) + census_rows = _rows( + conn, + f""" + SELECT raw_id, parser_fingerprint, status, member_count, detail + FROM raw_membership_census + WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + raw_ids, + ) + parser_census_rows = _rows( + conn, + f""" + SELECT raw_id, parser_fingerprint, status, logical_keys_json, detail + FROM raw_authority_parser_census + WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + raw_ids, + ) + logical_keys = tuple( + sorted( + { + str(value) + for row in (*source_rows, *membership_rows) + if (value := row.get("logical_source_key")) is not None + } + ) + ) + if logical_keys: + key_marks = ",".join("?" for _ in logical_keys) + head_rows = _rows( + conn, + f""" + SELECT logical_source_key, session_id, accepted_raw_id, + accepted_source_revision, hex(accepted_content_hash) AS accepted_content_hash, + accepted_frontier_kind, accepted_frontier, + acquisition_generation, append_end_offset + FROM index_tier.raw_revision_heads + WHERE logical_source_key IN ({key_marks}) ORDER BY logical_source_key + """, + logical_keys, + ) + else: + head_rows = [] + session_rows = _rows( + conn, + f""" + SELECT session_id, raw_id, hex(content_hash) AS content_hash, message_count + FROM index_tier.sessions + WHERE raw_id IN ({marks}) ORDER BY session_id + """, + raw_ids, + ) + authority_witness = json_document( + { + "parser_census": parser_census_rows, + "membership_census": census_rows, + "memberships": membership_rows, + "revision_heads": head_rows, + } + ) + source_preconditions = json_document( + {"raw_sessions": source_rows, "raw_authority_parser_census": parser_census_rows} + ) + index_preconditions = json_document({"sessions": session_rows, "revision_heads": head_rows}) + identity = { + "schema": "polylogue.raw-replay-plan.v2", + "input_raw_ids": list(raw_ids), + "logical_keys": list(logical_keys), + "authority_witness": authority_witness, + "source_preconditions": source_preconditions, + "index_preconditions": index_preconditions, + } + input_digest = _digest(identity) + return RawReplayPlan( + plan_id=f"raw-replay:{input_digest}", + input_digest=input_digest, + input_raw_ids=raw_ids, + logical_keys=logical_keys, + authority_witness=authority_witness, + source_preconditions=source_preconditions, + index_preconditions=index_preconditions, + ) + + +def build_raw_replay_plans(archive_root: Path, components: Sequence[tuple[str, ...]]) -> tuple[RawReplayPlan, ...]: + if not components: + return () + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: + conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + return tuple(build_raw_replay_plan(conn, component) for component in components) + + +def raw_replay_plan_last_attempts(archive_root: Path) -> dict[str, int]: + """Return durable attempt order; deleting ops.db cannot reset fairness.""" + source_db = archive_root / "source.db" + if not source_db.is_file(): + return {} + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='raw_authority_census_plans'" + ).fetchone() + if exists is None: + return {} + return { + str(row[0]): int(row[1]) + for row in conn.execute( + """ + SELECT cp.plan_id, MAX(cp.recorded_at_ms) + FROM raw_authority_census_plans AS cp + JOIN raw_authority_censuses AS c ON c.census_id = cp.census_id + WHERE cp.selected = 1 + AND c.lifecycle_status IN ('completed', 'interrupted') + GROUP BY cp.plan_id + """ + ) + } + + +def unresolved_raw_authority_blockers(archive_root: Path) -> int: + source_db = archive_root / "source.db" + if not source_db.is_file(): + return 0 + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as conn: + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='raw_authority_blockers'" + ).fetchone() + if exists is None: + return 0 + return int( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] + ) + + +def record_raw_authority_census( + archive_root: Path, + plans: Sequence[RawReplayPlan], + *, + selected_plan_ids: set[str], + executable_plan_ids: set[str] | None = None, + mode: str, + quiescent: bool, + scope: Mapping[str, object], + residual: Mapping[str, object], +) -> RawAuthorityCensusReceipt: + """Atomically publish a plan census, finalized only when no apply is pending.""" + if mode not in {"census", "dry_run", "apply"}: + raise ValueError(f"unsupported raw authority census mode: {mode}") + if mode != "apply" and selected_plan_ids: + raise ValueError(f"{mode} census cannot select plans for application") + now = int(time.time() * 1000) + inventory_digest = _digest([plan.plan_id for plan in plans]) + residual_digest = _digest(residual) + scope_json = _canonical_json(scope) + residual_json = _canonical_json(residual) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + previous = conn.execute( + """ + SELECT census_id, sequence_no, inventory_digest, residual_digest, + executable_plan_count, scope_json, parser_fingerprint, + mode, lifecycle_status, quiescent + FROM raw_authority_censuses ORDER BY sequence_no DESC LIMIT 1 + """ + ).fetchone() + sequence_no = int(previous[1]) + 1 if previous is not None else 1 + predecessor = str(previous[0]) if previous is not None else None + executable_ids = selected_plan_ids if executable_plan_ids is None else executable_plan_ids + unknown_ids = (selected_plan_ids | executable_ids) - {plan.plan_id for plan in plans} + if unknown_ids: + raise RuntimeError(f"raw authority census references unknown plans: {sorted(unknown_ids)}") + executable_count = len(executable_ids) + residual_count = len(plans) - executable_count + fixed_point = bool( + previous is not None + and int(previous[4]) == 0 + and executable_count == 0 + and str(previous[2]) == inventory_digest + and str(previous[3]) == residual_digest + and str(previous[5]) == scope_json + and str(previous[6]) == RAW_AUTHORITY_PARSER_FINGERPRINT + and str(previous[7]) == "dry_run" + and str(previous[8]) == "completed" + and bool(previous[9]) + and mode == "dry_run" + and quiescent + ) + census_id = f"census:{sequence_no}:{inventory_digest[:16]}:{residual_digest[:16]}" + lifecycle_status = "planned" if mode == "apply" and selected_plan_ids else "completed" + completed_at_ms = None if lifecycle_status == "planned" else now + post_inventory_digest = None if lifecycle_status == "planned" else inventory_digest + post_residual_json = None if lifecycle_status == "planned" else residual_json + post_residual_digest = None if lifecycle_status == "planned" else residual_digest + post_plan_count = None if lifecycle_status == "planned" else len(plans) + postflight_at_ms = None if lifecycle_status == "planned" else now + for plan in plans: + values = ( + plan.plan_id, + plan.input_digest, + _canonical_json(list(plan.input_raw_ids)), + _canonical_json(list(plan.logical_keys)), + _canonical_json(plan.authority_witness), + _canonical_json(plan.source_preconditions), + _canonical_json(plan.index_preconditions), + now, + ) + conn.execute( + """ + INSERT INTO raw_authority_plans ( + plan_id, input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(plan_id) DO NOTHING + """, + values, + ) + stored = conn.execute( + """ + SELECT input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json + FROM raw_authority_plans WHERE plan_id = ? + """, + (plan.plan_id,), + ).fetchone() + if stored != values[1:7]: + raise RuntimeError(f"immutable raw replay plan collision: {plan.plan_id}") + conn.execute( + """ + INSERT INTO raw_authority_censuses ( + census_id, sequence_no, scope_json, residual_json, + parser_fingerprint, mode, lifecycle_status, quiescent, + inventory_digest, residual_digest, plan_count, + post_inventory_digest, post_residual_json, + post_residual_digest, post_plan_count, postflight_at_ms, + executable_plan_count, residual_plan_count, + predecessor_census_id, fixed_point, created_at_ms, completed_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + census_id, + sequence_no, + scope_json, + residual_json, + RAW_AUTHORITY_PARSER_FINGERPRINT, + mode, + lifecycle_status, + int(quiescent), + inventory_digest, + residual_digest, + len(plans), + post_inventory_digest, + post_residual_json, + post_residual_digest, + post_plan_count, + postflight_at_ms, + executable_count, + residual_count, + predecessor, + int(fixed_point), + now, + completed_at_ms, + ), + ) + for ordinal, plan in enumerate(plans): + selected = plan.plan_id in selected_plan_ids + conn.execute( + """ + INSERT INTO raw_authority_census_plans ( + census_id, plan_id, ordinal, selected, outcome_status, + reason, next_action, application_receipt_json, + outcome_recorded, recorded_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, '{}', ?, ?) + """, + ( + census_id, + plan.plan_id, + ordinal, + int(selected), + RawReplayPlanStatus.RETRYABLE.value if selected else RawReplayPlanStatus.CARRIED_FORWARD.value, + "selected plan application is pending" + if selected + else "bounded scheduler carried this complete plan forward unchanged", + "execute this plan in the current pass" if selected else "retain for a later bounded pass", + 0 if selected else 1, + now, + ), + ) + if lifecycle_status != "planned": + conn.executemany( + """ + INSERT INTO raw_authority_census_post_plans (census_id, plan_id, ordinal) + VALUES (?, ?, ?) + """, + ((census_id, plan.plan_id, ordinal) for ordinal, plan in enumerate(plans)), + ) + return RawAuthorityCensusReceipt( + census_id=census_id, + sequence_no=sequence_no, + inventory_digest=inventory_digest, + residual_digest=residual_digest, + plan_count=len(plans), + executable_plan_count=executable_count, + residual_plan_count=residual_count, + post_inventory_digest=post_inventory_digest, + post_residual_digest=post_residual_digest, + post_plan_count=post_plan_count, + predecessor_census_id=predecessor, + mode=mode, + lifecycle_status=lifecycle_status, + quiescent=quiescent, + fixed_point=fixed_point, + ) + + +def validate_raw_replay_plan(archive_root: Path, plan: RawReplayPlan) -> tuple[bool, JSONDocument]: + try: + observed = build_raw_replay_plans(archive_root, (plan.input_raw_ids,))[0] + except Exception as exc: + logger.warning("raw replay plan validation could not rebuild %s", plan.plan_id, exc_info=True) + return False, json_document({"error": f"{type(exc).__name__}: {exc}"}) + return observed == plan, observed.to_dict() + + +def raw_replay_application_receipt(archive_root: Path, plan: RawReplayPlan) -> JSONDocument: + marks = ",".join("?" for _ in plan.input_raw_ids) + with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: + conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + source = _rows( + conn, + f""" + SELECT raw_id, parsed_at_ms, parse_error + FROM raw_sessions WHERE raw_id IN ({marks}) ORDER BY raw_id + """, + plan.input_raw_ids, + ) + memberships = _rows( + conn, + f""" + SELECT raw_id, logical_source_key, decision, decided_at_ms + FROM raw_session_memberships + WHERE raw_id IN ({marks}) ORDER BY raw_id, logical_source_key + """, + plan.input_raw_ids, + ) + applications = _rows( + conn, + f""" + SELECT decision_id, raw_id, session_id, logical_source_key, decision, + accepted_raw_id, hex(accepted_content_hash) AS accepted_content_hash, + decided_at_ms + FROM index_tier.raw_revision_applications + WHERE raw_id IN ({marks}) ORDER BY raw_id, decision_id + """, + plan.input_raw_ids, + ) + if plan.logical_keys: + key_marks = ",".join("?" for _ in plan.logical_keys) + heads = _rows( + conn, + f""" + SELECT logical_source_key, session_id, accepted_raw_id, + accepted_source_revision, + hex(accepted_content_hash) AS accepted_content_hash, + accepted_frontier_kind, accepted_frontier + FROM index_tier.raw_revision_heads + WHERE logical_source_key IN ({key_marks}) + ORDER BY logical_source_key + """, + plan.logical_keys, + ) + sessions = _rows( + conn, + f""" + SELECT s.session_id, s.raw_id, hex(s.content_hash) AS content_hash, + s.message_count + FROM index_tier.sessions AS s + JOIN index_tier.raw_revision_heads AS h ON h.session_id = s.session_id + WHERE h.logical_source_key IN ({key_marks}) + ORDER BY s.session_id + """, + plan.logical_keys, + ) + else: + heads = [] + sessions = [] + return json_document( + { + "schema": "polylogue.raw-replay-application-receipt.v2", + "source_rows": source, + "membership_rows": memberships, + "application_rows": applications, + "head_rows": heads, + "session_rows": sessions, + } + ) + + +def validate_raw_replay_application_receipt( + plan: RawReplayPlan, + receipt: Mapping[str, object], +) -> tuple[bool, tuple[str, ...]]: + """Prove exact replay postconditions; parsed timestamps are never sufficient.""" + problems: list[str] = [] + if receipt.get("schema") != "polylogue.raw-replay-application-receipt.v2": + problems.append("application receipt schema is not v2") + + def rows(name: str) -> list[Mapping[str, object]]: + value = receipt.get(name) + if not isinstance(value, list) or any(not isinstance(row, dict) for row in value): + problems.append(f"{name} is not a row list") + return [] + return value + + source_rows = rows("source_rows") + membership_rows = rows("membership_rows") + application_rows = rows("application_rows") + head_rows = rows("head_rows") + session_rows = rows("session_rows") + source_ids = {str(row.get("raw_id")) for row in source_rows} + if source_ids != set(plan.input_raw_ids): + problems.append("source receipt raw ids do not match the immutable plan") + if any(row.get("parsed_at_ms") is None or row.get("parse_error") is not None for row in source_rows): + problems.append("source receipt contains an unparsed or parse-failed raw") + expected_keys = set(plan.logical_keys) + head_keys = {str(row.get("logical_source_key")) for row in head_rows} + if not expected_keys: + problems.append("executed replay plan has no logical authority keys") + elif head_keys != expected_keys: + problems.append("accepted head keys do not match the immutable plan") + input_raw_ids = set(plan.input_raw_ids) + witness = plan.authority_witness.get("memberships") + expected_memberships = ( + {(str(row.get("raw_id")), str(row.get("logical_source_key"))) for row in witness if isinstance(row, dict)} + if isinstance(witness, list) + else set() + ) + observed_memberships = {(str(row.get("raw_id")), str(row.get("logical_source_key"))) for row in membership_rows} + if observed_memberships != expected_memberships: + problems.append("membership receipt pairs do not match the immutable authority witness") + terminal_membership_decisions = {"applied", "superseded_equivalent", "superseded_prefix"} + if any(row.get("decision") not in terminal_membership_decisions for row in membership_rows): + problems.append("membership receipt contains a non-terminal decision") + terminal_application_decisions = {"selected_baseline", "applied_append", "superseded"} + application_pairs = {(str(row.get("raw_id")), str(row.get("logical_source_key"))) for row in application_rows} + if any(row.get("decision") not in terminal_application_decisions for row in application_rows): + problems.append("application receipt contains a non-terminal decision") + if any(raw_id not in input_raw_ids or key not in expected_keys for raw_id, key in application_pairs): + problems.append("application receipt contains authority outside the immutable component") + terminal_keys = {key for _, key in observed_memberships | application_pairs} + if terminal_keys != expected_keys: + problems.append("terminal receipt keys do not exactly match the immutable plan") + head_session_ids = {str(row.get("session_id")) for row in head_rows} + session_ids = {str(row.get("session_id")) for row in session_rows} + if head_session_ids != session_ids: + problems.append("accepted head sessions do not match materialized session rows") + session_content = {str(row.get("session_id")): str(row.get("content_hash")) for row in session_rows} + if any( + str(row.get("accepted_content_hash")) != session_content.get(str(row.get("session_id"))) for row in head_rows + ): + problems.append("accepted head content hashes do not match materialized sessions") + if any(str(row.get("accepted_raw_id")) not in input_raw_ids for row in head_rows): + problems.append("accepted heads do not point into the immutable input component") + heads_by_key = {str(row.get("logical_source_key")): row for row in head_rows} + if len(heads_by_key) != len(head_rows): + problems.append("accepted head receipt contains duplicate logical authority keys") + sessions_by_id: dict[str, Mapping[str, object]] = {} + for session in session_rows: + session_id = str(session.get("session_id")) + existing_session = sessions_by_id.setdefault(session_id, session) + if existing_session != session: + problems.append(f"materialized session receipt conflicts for {session_id}") + application_keys: set[str] = set() + applications_matching_current_head: set[str] = set() + for application in application_rows: + key = str(application.get("logical_source_key")) + application_keys.add(key) + head = heads_by_key.get(key) + if head is None: + problems.append(f"application receipt has no accepted head for {key}") + continue + application_authority = ( + str(application.get("session_id")), + str(application.get("accepted_raw_id")), + str(application.get("accepted_content_hash")), + ) + head_authority = ( + str(head.get("session_id")), + str(head.get("accepted_raw_id")), + str(head.get("accepted_content_hash")), + ) + if application_authority == head_authority: + applications_matching_current_head.add(key) + materialized_session = sessions_by_id.get(str(head.get("session_id"))) + if materialized_session is None: + continue + if str(materialized_session.get("raw_id")) != str(head.get("accepted_raw_id")) or str( + materialized_session.get("content_hash") + ) != str(head.get("accepted_content_hash")): + problems.append(f"materialized session authority does not match the head for {key}") + for key in sorted(application_keys - applications_matching_current_head): + problems.append(f"no application accepted authority matches the current head for {key}") + return not problems, tuple(problems) + + +def record_raw_replay_outcome( + archive_root: Path, + census_id: str, + outcome: RawReplayPlanOutcome, +) -> None: + now = int(time.time() * 1000) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + updated = conn.execute( + """ + UPDATE raw_authority_census_plans + SET outcome_status = ?, reason = ?, next_action = ?, + application_receipt_json = ?, outcome_recorded = 1, + recorded_at_ms = ? + WHERE census_id = ? AND plan_id = ? AND selected = 1 + AND outcome_recorded = 0 + """, + ( + outcome.status.value, + outcome.reason, + outcome.next_action, + _canonical_json(outcome.application_receipt or {}), + now, + census_id, + outcome.plan_id, + ), + ).rowcount + if updated != 1: + raise RuntimeError(f"outcome does not conserve one selected plan: {outcome.plan_id}") + + +def _raw_replay_plan_from_row(row: sqlite3.Row) -> RawReplayPlan: + return RawReplayPlan( + plan_id=str(row["plan_id"]), + input_digest=str(row["input_digest"]), + input_raw_ids=tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))), + logical_keys=tuple(str(value) for value in json.loads(str(row["logical_keys_json"]))), + authority_witness=json_document(json.loads(str(row["authority_witness_json"]))), + source_preconditions=json_document(json.loads(str(row["source_preconditions_json"]))), + index_preconditions=json_document(json.loads(str(row["index_preconditions_json"]))), + ) + + +def _raw_authority_census_receipt(conn: sqlite3.Connection, census_id: str) -> RawAuthorityCensusReceipt: + row = conn.execute( + """ + SELECT census_id, sequence_no, inventory_digest, residual_digest, + plan_count, executable_plan_count, residual_plan_count, + post_inventory_digest, post_residual_digest, post_plan_count, + predecessor_census_id, mode, lifecycle_status, quiescent, + fixed_point + FROM raw_authority_censuses WHERE census_id = ? + """, + (census_id,), + ).fetchone() + if row is None: + raise KeyError(census_id) + return RawAuthorityCensusReceipt( + census_id=str(row[0]), + sequence_no=int(row[1]), + inventory_digest=str(row[2]), + residual_digest=str(row[3]), + plan_count=int(row[4]), + executable_plan_count=int(row[5]), + residual_plan_count=int(row[6]), + post_inventory_digest=str(row[7]) if row[7] is not None else None, + post_residual_digest=str(row[8]) if row[8] is not None else None, + post_plan_count=int(row[9]) if row[9] is not None else None, + predecessor_census_id=str(row[10]) if row[10] is not None else None, + mode=str(row[11]), + lifecycle_status=str(row[12]), + quiescent=bool(row[13]), + fixed_point=bool(row[14]), + ) + + +def finalize_raw_authority_census( + archive_root: Path, + census_id: str, + *, + post_plans: Sequence[RawReplayPlan], + post_residual: Mapping[str, object], + interrupted: bool = False, +) -> RawAuthorityCensusReceipt: + """Publish a census only after every selected plan has a recorded outcome.""" + now = int(time.time() * 1000) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + status = conn.execute( + "SELECT lifecycle_status FROM raw_authority_censuses WHERE census_id = ?", + (census_id,), + ).fetchone() + if status is None: + raise KeyError(census_id) + if str(status[0]) != "planned": + return _raw_authority_census_receipt(conn, census_id) + pending = int( + conn.execute( + """ + SELECT COUNT(*) FROM raw_authority_census_plans + WHERE census_id = ? AND selected = 1 AND outcome_recorded = 0 + """, + (census_id,), + ).fetchone()[0] + ) + if pending: + raise RuntimeError(f"raw authority census still has {pending} pending selected outcome(s)") + post_ids = {plan.plan_id for plan in post_plans} + persistent = { + str(row[0]) + for row in conn.execute( + """ + SELECT plan_id FROM raw_authority_census_plans + WHERE census_id = ? AND outcome_status IN ('retryable', 'carried_forward') + """, + (census_id,), + ) + } + if not persistent.issubset(post_ids): + raise RuntimeError( + f"raw authority postflight changed a retryable/carried-forward plan: {sorted(persistent - post_ids)}" + ) + for plan in post_plans: + values = ( + plan.plan_id, + plan.input_digest, + _canonical_json(list(plan.input_raw_ids)), + _canonical_json(list(plan.logical_keys)), + _canonical_json(plan.authority_witness), + _canonical_json(plan.source_preconditions), + _canonical_json(plan.index_preconditions), + now, + ) + conn.execute( + """ + INSERT INTO raw_authority_plans ( + plan_id, input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(plan_id) DO NOTHING + """, + values, + ) + stored = conn.execute( + """ + SELECT input_digest, input_raw_ids_json, logical_keys_json, + authority_witness_json, source_preconditions_json, + index_preconditions_json + FROM raw_authority_plans WHERE plan_id = ? + """, + (plan.plan_id,), + ).fetchone() + if stored != values[1:7]: + raise RuntimeError(f"immutable raw replay postflight plan collision: {plan.plan_id}") + conn.executemany( + """ + INSERT INTO raw_authority_census_post_plans (census_id, plan_id, ordinal) + VALUES (?, ?, ?) + """, + ((census_id, plan.plan_id, ordinal) for ordinal, plan in enumerate(post_plans)), + ) + post_inventory_digest = _digest([plan.plan_id for plan in post_plans]) + post_residual_json = _canonical_json(post_residual) + post_residual_digest = _digest(post_residual) + conn.execute( + """ + UPDATE raw_authority_censuses + SET lifecycle_status = ?, completed_at_ms = ?, + post_inventory_digest = ?, post_residual_json = ?, + post_residual_digest = ?, post_plan_count = ?, postflight_at_ms = ? + WHERE census_id = ? AND lifecycle_status = 'planned' + """, + ( + "interrupted" if interrupted else "completed", + now, + post_inventory_digest, + post_residual_json, + post_residual_digest, + len(post_plans), + now, + census_id, + ), + ) + return _raw_authority_census_receipt(conn, census_id) + + +def recover_interrupted_raw_authority_censuses( + archive_root: Path, +) -> tuple[tuple[str, JSONDocument], ...]: + """Reconcile unfinished apply censuses from durable postconditions.""" + source_db = archive_root / "source.db" + if not source_db.is_file(): + return () + with closing(sqlite3.connect(source_db)) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + """ + SELECT c.census_id, p.* + FROM raw_authority_censuses AS c + JOIN raw_authority_census_plans AS cp ON cp.census_id = c.census_id + JOIN raw_authority_plans AS p ON p.plan_id = cp.plan_id + WHERE c.lifecycle_status = 'planned' + AND cp.selected = 1 AND cp.outcome_recorded = 0 + ORDER BY c.sequence_no, cp.ordinal + """ + ).fetchall() + census_scopes = tuple( + (str(row[0]), json_document(json.loads(str(row[1])))) + for row in conn.execute( + """ + SELECT census_id, scope_json + FROM raw_authority_censuses + WHERE lifecycle_status = 'planned' + ORDER BY sequence_no + """ + ) + ) + for row in rows: + census_id = str(row["census_id"]) + plan = _raw_replay_plan_from_row(row) + receipt = raw_replay_application_receipt(archive_root, plan) + valid_receipt, problems = validate_raw_replay_application_receipt(plan, receipt) + if valid_receipt: + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.EXECUTED, + "interrupted application recovered from exact durable postconditions", + "none", + receipt, + ) + record_raw_replay_outcome(archive_root, census_id, outcome) + continue + valid_plan, observed = validate_raw_replay_plan(archive_root, plan) + if not valid_plan: + reject_stale_raw_replay_plan(archive_root, census_id, plan, observed) + else: + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.RETRYABLE, + "interrupted before exact application postconditions were durable: " + "; ".join(problems), + "retry the same immutable plan", + receipt, + ) + record_raw_replay_outcome(archive_root, census_id, outcome) + return census_scopes + + +def resolve_raw_authority_blocker(archive_root: Path, blocker_id: str, *, resolution: str) -> JSONDocument: + """Explicitly acknowledge current evidence and reopen replanning.""" + if not resolution.strip(): + raise ValueError("raw authority blocker resolution must be non-empty") + source_db = archive_root / "source.db" + with closing(sqlite3.connect(source_db)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + """ + SELECT b.blocker_id, b.plan_id, b.census_id, b.expected_json, p.input_raw_ids_json + FROM raw_authority_blockers AS b + JOIN raw_authority_plans AS p ON p.plan_id = b.plan_id + WHERE b.blocker_id = ? AND b.resolved_at_ms IS NULL + """, + (blocker_id,), + ).fetchone() + if row is None: + conn.rollback() + raise KeyError(blocker_id) + input_raw_ids = tuple(str(value) for value in json.loads(str(row["input_raw_ids_json"]))) + observed = build_raw_replay_plan(conn, input_raw_ids) + now = int(time.time() * 1000) + full_receipt = json_document( + { + "schema": "polylogue.raw-authority-blocker-resolution.v1", + "blocker_id": blocker_id, + "superseded_plan_id": str(row["plan_id"]), + "current_plan": observed.to_dict(), + "operator_resolution": resolution.strip(), + "resolved_at_ms": now, + } + ) + updated = conn.execute( + """ + UPDATE raw_authority_blockers + SET resolved_at_ms = ?, resolution = ? + WHERE blocker_id = ? AND resolved_at_ms IS NULL + """, + (now, _canonical_json(full_receipt), blocker_id), + ).rowcount + if updated != 1: + conn.rollback() + raise RuntimeError(f"raw authority blocker changed during resolution: {blocker_id}") + conn.commit() + return json_document( + { + "schema": "polylogue.raw-authority-blocker-resolution-summary.v1", + "blocker_id": blocker_id, + "superseded_plan_id": str(row["plan_id"]), + "current_plan": { + "plan_id": observed.plan_id, + "input_digest": observed.input_digest, + "input_raw_count": len(observed.input_raw_ids), + "logical_key_count": len(observed.logical_keys), + }, + "operator_resolution": resolution.strip(), + "resolved_at_ms": now, + "detail_query_handle": raw_authority_detail_query_handle(str(row["census_id"]), str(row["plan_id"])), + } + ) + + +def reject_stale_raw_replay_plan( + archive_root: Path, + census_id: str, + plan: RawReplayPlan, + observed: JSONDocument, +) -> RawReplayPlanOutcome: + """Persist the fail-closed blocker before returning observational output.""" + now = int(time.time() * 1000) + blocker_id = f"raw-authority-blocker:{_digest([census_id, plan.plan_id, observed])}" + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.REJECTED_STALE, + "immutable source/index preconditions changed after the census", + "resolve the durable raw-authority blocker before automatic convergence resumes", + json_document({"expected": plan.to_dict(), "observed": observed, "blocker_id": blocker_id}), + ) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + conn.execute( + """ + INSERT INTO raw_authority_blockers ( + blocker_id, plan_id, census_id, reason, expected_json, + observed_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(blocker_id) DO NOTHING + """, + ( + blocker_id, + plan.plan_id, + census_id, + outcome.reason, + _canonical_json(plan.to_dict()), + _canonical_json(observed), + now, + ), + ) + updated = conn.execute( + """ + UPDATE raw_authority_census_plans + SET outcome_status = 'rejected_stale', reason = ?, next_action = ?, + application_receipt_json = ?, outcome_recorded = 1, + recorded_at_ms = ? + WHERE census_id = ? AND plan_id = ? AND selected = 1 + AND outcome_recorded = 0 + """, + ( + outcome.reason, + outcome.next_action, + _canonical_json(outcome.application_receipt or {}), + now, + census_id, + plan.plan_id, + ), + ).rowcount + if updated != 1: + raise RuntimeError(f"stale rejection does not conserve one selected plan: {plan.plan_id}") + open_count = conn.execute( + "SELECT COUNT(*) FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (plan.plan_id,), + ).fetchone()[0] + if int(open_count) != 1: + raise RuntimeError(f"stale rejection did not leave one open blocker: {plan.plan_id}") + return outcome + + +def reject_invalid_raw_replay_application( + archive_root: Path, + census_id: str, + plan: RawReplayPlan, + receipt: JSONDocument, + problems: Sequence[str], +) -> RawReplayPlanOutcome: + """Fail closed when a writer returns without exact application postconditions.""" + now = int(time.time() * 1000) + observed = json_document({"application_receipt": receipt, "problems": list(problems)}) + blocker_id = f"raw-authority-blocker:{_digest([census_id, plan.plan_id, observed])}" + outcome = RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.REJECTED_STALE, + "raw replay application did not satisfy exact durable postconditions", + "resolve the durable raw-authority blocker before automatic convergence resumes", + json_document({"expected": plan.to_dict(), "observed": observed, "blocker_id": blocker_id}), + ) + with closing(sqlite3.connect(archive_root / "source.db")) as conn, conn: + conn.execute( + """ + INSERT INTO raw_authority_blockers ( + blocker_id, plan_id, census_id, reason, expected_json, + observed_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(blocker_id) DO NOTHING + """, + ( + blocker_id, + plan.plan_id, + census_id, + outcome.reason, + _canonical_json(plan.to_dict()), + _canonical_json(observed), + now, + ), + ) + updated = conn.execute( + """ + UPDATE raw_authority_census_plans + SET outcome_status = 'rejected_stale', reason = ?, next_action = ?, + application_receipt_json = ?, outcome_recorded = 1, + recorded_at_ms = ? + WHERE census_id = ? AND plan_id = ? AND selected = 1 + AND outcome_recorded = 0 + """, + ( + outcome.reason, + outcome.next_action, + _canonical_json(outcome.application_receipt or {}), + now, + census_id, + plan.plan_id, + ), + ).rowcount + if updated != 1: + raise RuntimeError(f"invalid application does not conserve one selected plan: {plan.plan_id}") + open_count = conn.execute( + "SELECT COUNT(*) FROM raw_authority_blockers WHERE plan_id = ? AND resolved_at_ms IS NULL", + (plan.plan_id,), + ).fetchone()[0] + if int(open_count) != 1: + raise RuntimeError(f"invalid application did not leave one open blocker: {plan.plan_id}") + return outcome + + +__all__ = [ + "RAW_AUTHORITY_CENSUS_QUERY_PREFIX", + "RAW_AUTHORITY_DETAIL_CHUNK_CHARS", + "RAW_AUTHORITY_DETAIL_QUERY_PREFIX", + "RAW_AUTHORITY_PARSER_FINGERPRINT", + "RawAuthorityCensusReceipt", + "RawReplayPlan", + "RawReplayPlanOutcome", + "RawReplayPlanStatus", + "build_raw_replay_plan", + "build_raw_replay_plans", + "finalize_raw_authority_census", + "raw_replay_application_receipt", + "raw_authority_census_query_handle", + "raw_authority_detail_query_handle", + "raw_replay_plan_last_attempts", + "recover_interrupted_raw_authority_censuses", + "read_raw_authority_census", + "read_raw_authority_detail", + "record_raw_authority_census", + "record_raw_replay_outcome", + "reject_invalid_raw_replay_application", + "reject_stale_raw_replay_plan", + "resolve_raw_authority_blocker", + "unresolved_raw_authority_blockers", + "validate_raw_replay_plan", + "validate_raw_replay_application_receipt", +] diff --git a/polylogue/storage/repair.py b/polylogue/storage/repair.py index d9391722e0..938554a19f 100644 --- a/polylogue/storage/repair.py +++ b/polylogue/storage/repair.py @@ -14,7 +14,6 @@ from contextlib import closing, contextmanager, suppress from dataclasses import dataclass, field from datetime import UTC, datetime -from enum import StrEnum from pathlib import Path from typing import cast @@ -57,12 +56,32 @@ count_messages_by_type_sync, count_unclassified_message_type_sync, ) +from polylogue.storage.raw_authority import ( + RawAuthorityCensusReceipt, + RawReplayPlan, + RawReplayPlanOutcome, + RawReplayPlanStatus, + build_raw_replay_plans, + finalize_raw_authority_census, + raw_replay_application_receipt, + raw_replay_plan_last_attempts, + record_raw_authority_census, + record_raw_replay_outcome, + recover_interrupted_raw_authority_censuses, + reject_invalid_raw_replay_application, + reject_stale_raw_replay_plan, + unresolved_raw_authority_blockers, + validate_raw_replay_application_receipt, + validate_raw_replay_plan, +) logger = get_logger(__name__) _MAINTENANCE_TARGET_CATALOG = build_maintenance_target_catalog() _PROBE_ONLY_EXACT_MESSAGE_ROW_LIMIT = 100_000 RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES = 1024 * 1024 * 1024 RAW_MATERIALIZATION_RESOURCE_BLOCK_REASON = "non-stream-safe raw payload exceeds the bounded replay limit" +RAW_MATERIALIZATION_CENSUS_COMPONENT_LIMIT = 25 +RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT = 8 _TRANSIENT_LOCK_PARSE_ERROR = "OperationalError: database is locked" _QUARANTINED_ACCEPTED_RAW_REPAIR_DETAIL = "repair:accepted_quarantined_raw_exact_byte_and_semantic_proof" _QUARANTINED_ACCEPTED_RAW_REPAIR_LIMIT = 100 @@ -4578,6 +4597,12 @@ class RawMaterializationCandidates: expanded_raw_ids: tuple[str, ...] = () expanded_blob_bytes: dict[str, int] = field(default_factory=dict) authority_components: tuple[tuple[str, ...], ...] = () + missing_blob_raw_ids: tuple[str, ...] = () + adoption_deferred_raw_ids: tuple[str, ...] = () + authority_quarantined_raw_ids: tuple[str, ...] = () + byte_authority_fragment_raw_ids: tuple[str, ...] = () + byte_authority_quarantined_raw_ids: tuple[str, ...] = () + byte_authority_pending_raw_ids: tuple[str, ...] = () @property def total_blob_bytes(self) -> int: @@ -4588,34 +4613,6 @@ def max_blob_bytes(self) -> int: return max((self.raw_blob_bytes.get(raw_id, 0) for raw_id in self.raw_ids), default=0) -class RawReplayPlanStatus(StrEnum): - EXECUTED = "executed" - RETRYABLE = "retryable" - DEFERRED = "deferred" - TERMINAL = "terminal" - REJECTED_STALE = "rejected_stale" - - -@dataclass(frozen=True, slots=True) -class RawReplayPlanOutcome: - plan_id: str - input_raw_ids: tuple[str, ...] - status: RawReplayPlanStatus - reason: str - next_action: str - - def to_dict(self) -> JSONDocument: - return json_document( - { - "plan_id": self.plan_id, - "input_raw_ids": list(self.input_raw_ids), - "status": self.status.value, - "reason": self.reason, - "next_action": self.next_action, - } - ) - - def _raw_materialization_origin_from_provider(provider: str | None) -> str | None: if provider is None: return None @@ -4672,6 +4669,12 @@ def _raw_materialization_candidate_ids( missing_blob_source_available = 0 missing_blob_source_missing = 0 already_parsed = 0 + missing_blob_raw_ids: list[str] = [] + adoption_deferred_raw_ids: list[str] = [] + authority_quarantined_raw_ids: list[str] = [] + byte_authority_fragment_raw_ids: list[str] = [] + byte_authority_quarantined_raw_ids: list[str] = [] + byte_authority_pending_raw_ids: list[str] = [] expanded_raw_ids: tuple[str, ...] = () expanded_blob_bytes: dict[str, int] = {} authority_components: tuple[tuple[str, ...], ...] = () @@ -4796,8 +4799,10 @@ def _raw_materialization_candidate_ids( byte_authority_quarantined = 0 byte_authority_pending = 0 for row in rows: + row_raw_id = str(row["raw_id"]) if bool(row["adoption_deferred"]): adoption_deferred += 1 + adoption_deferred_raw_ids.append(row_raw_id) continue if bool(row["application_terminal"]): continue @@ -4805,15 +4810,19 @@ def _raw_materialization_candidate_ids( continue if bool(row["membership_authority_quarantined"]): authority_quarantined += 1 + authority_quarantined_raw_ids.append(row_raw_id) continue if bool(row["byte_authority_fragment"]): byte_authority_fragments += 1 + byte_authority_fragment_raw_ids.append(row_raw_id) continue if bool(row["byte_authority_quarantined"]): byte_authority_quarantined += 1 + byte_authority_quarantined_raw_ids.append(row_raw_id) continue if bool(row["byte_authority_pending"]): byte_authority_pending += 1 + byte_authority_pending_raw_ids.append(row_raw_id) continue if row["parse_error"] and not _raw_materialization_retryable_missing_blob_error(row["parse_error"]): continue @@ -4835,6 +4844,7 @@ def _raw_materialization_candidate_ids( already_parsed += 1 else: missing_blobs += 1 + missing_blob_raw_ids.append(row_raw_id) if _raw_materialization_source_available(str(row["source_path"] or "")): missing_blob_source_available += 1 else: @@ -4870,6 +4880,12 @@ def _raw_materialization_candidate_ids( expanded_raw_ids=expanded_raw_ids, expanded_blob_bytes=expanded_blob_bytes, authority_components=authority_components, + missing_blob_raw_ids=tuple(sorted(missing_blob_raw_ids)), + adoption_deferred_raw_ids=tuple(sorted(adoption_deferred_raw_ids)), + authority_quarantined_raw_ids=tuple(sorted(authority_quarantined_raw_ids)), + byte_authority_fragment_raw_ids=tuple(sorted(byte_authority_fragment_raw_ids)), + byte_authority_quarantined_raw_ids=tuple(sorted(byte_authority_quarantined_raw_ids)), + byte_authority_pending_raw_ids=tuple(sorted(byte_authority_pending_raw_ids)), ) @@ -4896,7 +4912,9 @@ def _raw_materialization_ordered_components( candidate_ids = set(candidates.raw_ids) source_components = candidates.authority_components or tuple((raw_id,) for raw_id in candidates.raw_ids) components = [component for component in source_components if candidate_ids.intersection(component)] - last_attempts = _raw_replay_plan_last_attempts(archive_root) + plans = build_raw_replay_plans(archive_root, components) + plan_ids = {plan.input_raw_ids: plan.plan_id for plan in plans} + last_attempts = raw_replay_plan_last_attempts(archive_root) def candidate_order(raw_id: str) -> tuple[int, int, str]: if raw_id in candidates.raw_acquired_at_ms: @@ -4906,8 +4924,8 @@ def candidate_order(raw_id: str) -> tuple[int, int, str]: return sorted( components, key=lambda component: ( - _raw_replay_plan_id(component) in last_attempts, - last_attempts.get(_raw_replay_plan_id(component), 0), + plan_ids[component] in last_attempts, + last_attempts.get(plan_ids[component], 0), min(candidate_order(raw_id) for raw_id in component if raw_id in candidate_ids), component, ), @@ -4933,50 +4951,114 @@ def _raw_materialization_component_blob_bytes(candidates: RawMaterializationCand return candidates.expanded_blob_bytes.get(raw_id, candidates.raw_blob_bytes.get(raw_id, 0)) -def _raw_replay_plan_id(component: tuple[str, ...]) -> str: - digest = hashlib.sha256() - digest.update(b"polylogue.raw-replay-plan.v1\0") - for raw_id in sorted(component): - digest.update(raw_id.encode("utf-8")) - digest.update(b"\0") - return f"raw-replay:{digest.hexdigest()}" +def _raw_authority_scope( + *, + raw_artifact_id: str | None, + provider: str | None, + source_family: str | None, + source_root: Path | None, + raw_artifact_limit: int | None, +) -> dict[str, object]: + return { + "raw_artifact_id": raw_artifact_id, + "provider": provider, + "source_family": source_family, + "source_root": str(source_root) if source_root is not None else None, + "raw_artifact_limit": raw_artifact_limit, + } -def _raw_replay_plan_last_attempts(archive_root: Path) -> dict[str, int]: - """Read disposable daemon receipts to rotate retryable plans fairly.""" - ops_db = archive_root / "ops.db" - if not ops_db.is_file(): - return {} - try: - with closing(sqlite3.connect(f"file:{ops_db}?mode=ro", uri=True)) as conn: - exists = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='daemon_events'").fetchone() - if exists is None: - return {} - rows = conn.execute( - """ - SELECT json_extract(outcome.value, '$.plan_id'), MAX(event.id) - FROM daemon_events AS event, - json_each(event.payload_json, '$.plan_outcomes') AS outcome - WHERE event.kind = 'raw_materialization_pass' - AND json_valid(event.payload_json) - AND json_extract(outcome.value, '$.plan_id') IS NOT NULL - GROUP BY json_extract(outcome.value, '$.plan_id') - """ - ) - return {str(row[0]): int(row[1]) for row in rows} - except sqlite3.Error: - logger.warning("raw replay: failed to read prior plan attempts", exc_info=True) - return {} +def _raw_authority_residual( + candidates: RawMaterializationCandidates, + *, + census_pending_raw_ids: tuple[str, ...] = (), + resource_blocked_plan_ids: tuple[str, ...] = (), +) -> dict[str, object]: + """Return identity-sensitive residual debt for fixed-point comparison.""" + census_pending_digest = hashlib.sha256( + json.dumps(list(census_pending_raw_ids), separators=(",", ":")).encode() + ).hexdigest() + return { + "missing_blob_raw_ids": list(candidates.missing_blob_raw_ids), + "adoption_deferred_raw_ids": list(candidates.adoption_deferred_raw_ids), + "authority_quarantined_raw_ids": list(candidates.authority_quarantined_raw_ids), + "byte_authority_fragment_raw_ids": list(candidates.byte_authority_fragment_raw_ids), + "byte_authority_quarantined_raw_ids": list(candidates.byte_authority_quarantined_raw_ids), + "byte_authority_pending_raw_ids": list(candidates.byte_authority_pending_raw_ids), + # A large initial catch-up may need hundreds of bounded census passes. + # Keep every progress receipt identity-sensitive without copying the + # entire shrinking raw-ID backlog into source.db on every pass. + "census_pending_raw_count": len(census_pending_raw_ids), + "census_pending_raw_digest": census_pending_digest, + "resource_blocked_plan_ids": list(resource_blocked_plan_ids), + } + + +def _raw_authority_postflight_snapshot( + archive_root: Path, + candidates: RawMaterializationCandidates, +) -> tuple[tuple[RawReplayPlan, ...], dict[str, object]]: + """Build the complete post-pass plan inventory and typed residual debt.""" + components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + plans = build_raw_replay_plans(archive_root, components) + blocked_plan_ids = tuple( + sorted( + plan.plan_id + for component, plan in zip(components, plans, strict=True) + if sum(_raw_materialization_component_blob_bytes(candidates, member) for member in component) + > RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + ) + ) + return plans, _raw_authority_residual(candidates, resource_blocked_plan_ids=blocked_plan_ids) + + +def _raw_authority_candidates_for_scope( + config: Config, + scope: Mapping[str, object], +) -> RawMaterializationCandidates: + source_root_value = scope.get("source_root") + return _raw_materialization_candidate_ids( + config, + raw_artifact_id=str(scope["raw_artifact_id"]) if scope.get("raw_artifact_id") is not None else None, + provider=str(scope["provider"]) if scope.get("provider") is not None else None, + source_family=str(scope["source_family"]) if scope.get("source_family") is not None else None, + source_root=Path(str(source_root_value)) if source_root_value is not None else None, + ) + + +def _raw_materialization_base_metrics( + candidates: RawMaterializationCandidates, + *, + recovered_census_count: int, +) -> dict[str, float]: + metrics = { + "raw_materialization_candidate_count": float(len(candidates.raw_ids)), + "raw_materialization_missing_blob_count": float(candidates.missing_blobs), + "raw_materialization_missing_blob_source_available_count": float(candidates.missing_blob_source_available), + "raw_materialization_missing_blob_source_missing_count": float(candidates.missing_blob_source_missing), + "raw_materialization_already_parsed_count": float(candidates.already_parsed), + "raw_materialization_total_blob_bytes": float(candidates.total_blob_bytes), + "raw_materialization_max_blob_bytes": float(candidates.max_blob_bytes), + "raw_materialization_adoption_deferred_count": float(candidates.adoption_deferred), + "raw_materialization_authority_quarantined_count": float(candidates.authority_quarantined), + "raw_materialization_byte_authority_fragment_count": float(candidates.byte_authority_fragments), + "raw_materialization_byte_authority_quarantined_count": float(candidates.byte_authority_quarantined), + "raw_materialization_byte_authority_pending_count": float(candidates.byte_authority_pending), + } + if recovered_census_count: + metrics["raw_materialization_recovered_census_count"] = float(recovered_census_count) + return metrics def _raw_replay_plan_outcome( conn: sqlite3.Connection, - component: tuple[str, ...], + plan: RawReplayPlan, *, remaining: RawMaterializationCandidates, ) -> RawReplayPlanOutcome: """Conserve one selected component into an explicit post-pass state.""" - plan_id = _raw_replay_plan_id(component) + plan_id = plan.plan_id + component = plan.input_raw_ids remaining_ids = set(remaining.expanded_raw_ids) | set(remaining.raw_ids) if remaining_ids.intersection(component): return RawReplayPlanOutcome( @@ -5067,16 +5149,37 @@ def _raw_replay_plan_outcome( def _raw_replay_plan_outcomes( archive_root: Path, - components: Sequence[tuple[str, ...]], + plans: Sequence[RawReplayPlan], *, remaining: RawMaterializationCandidates, ) -> tuple[RawReplayPlanOutcome, ...]: - if not components: + if not plans: return () with closing(sqlite3.connect(f"file:{archive_root / 'source.db'}?mode=ro", uri=True)) as conn: conn.row_factory = sqlite3.Row conn.execute("ATTACH DATABASE ? AS index_tier", (str(archive_root / "index.db"),)) - return tuple(_raw_replay_plan_outcome(conn, component, remaining=remaining) for component in components) + return tuple(_raw_replay_plan_outcome(conn, plan, remaining=remaining) for plan in plans) + + +def _raw_replay_conservation_metrics( + plans: Sequence[RawReplayPlan], + selected_plan_ids: set[str], + outcomes: Sequence[RawReplayPlanOutcome], +) -> tuple[int, int, int]: + """Return total plans, all carried inventory, and exact algebra errors.""" + outcome_ids = [outcome.plan_id for outcome in outcomes] + outcome_id_set = set(outcome_ids) + conservation_errors = ( + len(selected_plan_ids - outcome_id_set) + + len(outcome_id_set - selected_plan_ids) + + (len(outcome_ids) - len(outcome_id_set)) + ) + carried_forward = ( + len(plans) + - len(selected_plan_ids) + + sum(outcome.status is RawReplayPlanStatus.CARRIED_FORWARD for outcome in outcomes) + ) + return len(plans), carried_forward, conservation_errors def _raw_materialization_bucket_summary( @@ -5555,6 +5658,7 @@ class RepairResult: detail: str = "" metrics: dict[str, float] = field(default_factory=dict) plan_outcomes: tuple[RawReplayPlanOutcome, ...] = () + census_receipt: RawAuthorityCensusReceipt | None = None def to_dict(self) -> JSONDocument: payload: dict[str, object] = { @@ -5567,7 +5671,30 @@ def to_dict(self) -> JSONDocument: "metrics": dict(self.metrics), } if self.plan_outcomes: - payload["plan_outcomes"] = [outcome.to_dict() for outcome in self.plan_outcomes] + payload["plan_outcome_count"] = len(self.plan_outcomes) + payload["plan_outcomes"] = [ + outcome.to_summary_dict() for outcome in self.plan_outcomes[:RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT] + ] + payload["plan_outcomes_truncated"] = len(self.plan_outcomes) > RAW_MATERIALIZATION_OUTCOME_SAMPLE_LIMIT + if self.census_receipt is not None: + payload["census"] = { + "census_id": self.census_receipt.census_id, + "sequence_no": self.census_receipt.sequence_no, + "inventory_digest": self.census_receipt.inventory_digest, + "residual_digest": self.census_receipt.residual_digest, + "plan_count": self.census_receipt.plan_count, + "post_inventory_digest": self.census_receipt.post_inventory_digest, + "post_residual_digest": self.census_receipt.post_residual_digest, + "post_plan_count": self.census_receipt.post_plan_count, + "executable_plan_count": self.census_receipt.executable_plan_count, + "residual_plan_count": self.census_receipt.residual_plan_count, + "predecessor_census_id": self.census_receipt.predecessor_census_id, + "mode": self.census_receipt.mode, + "lifecycle_status": self.census_receipt.lifecycle_status, + "quiescent": self.census_receipt.quiescent, + "fixed_point": self.census_receipt.fixed_point, + "query_handle": self.census_receipt.query_handle, + } return json_document(payload) @@ -5766,6 +5893,7 @@ def _internal_derived_repair_result( detail: str, metrics: dict[str, float] | None = None, plan_outcomes: tuple[RawReplayPlanOutcome, ...] = (), + census_receipt: RawAuthorityCensusReceipt | None = None, ) -> RepairResult: return RepairResult( name=name, @@ -5776,6 +5904,7 @@ def _internal_derived_repair_result( detail=detail, metrics=dict(metrics or {}), plan_outcomes=plan_outcomes, + census_receipt=census_receipt, ) @@ -6381,6 +6510,33 @@ def repair_raw_materialization( progress_callback: ProgressCallback | None = None, ) -> RepairResult: """Converge retained raws through typed per-session revision authority.""" + archive_root = _raw_materialization_archive_root(config) + recovered_censuses = recover_interrupted_raw_authority_censuses(archive_root) + for recovered_census_id, recovered_scope in recovered_censuses: + recovered_candidates = _raw_authority_candidates_for_scope(config, recovered_scope) + recovered_post_plans, recovered_post_residual = _raw_authority_postflight_snapshot( + archive_root, + recovered_candidates, + ) + finalize_raw_authority_census( + archive_root, + recovered_census_id, + post_plans=recovered_post_plans, + post_residual=recovered_post_residual, + interrupted=True, + ) + recovered_census_count = len(recovered_censuses) + blocker_count = unresolved_raw_authority_blockers(archive_root) + if blocker_count: + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=( + f"Raw materialization is fail-closed behind {blocker_count:,} unresolved durable stale-plan blocker(s)" + ), + metrics={"raw_materialization_unresolved_blocker_count": float(blocker_count)}, + ) candidates = _raw_materialization_candidate_ids( config, raw_artifact_id=raw_artifact_id, @@ -6388,9 +6544,131 @@ def repair_raw_materialization( source_family=source_family, source_root=source_root, ) + from polylogue.sources.revision_backfill import ( + RawRevisionReplayResourceBlockedError, + census_historical_revision_evidence, + uncensused_historical_revision_raw_ids, + ) + + relevant_raw_ids = list(candidates.expanded_raw_ids or tuple(candidates.raw_ids)) + uncensused_raw_ids = set(uncensused_historical_revision_raw_ids(archive_root, relevant_raw_ids)) + census_failed_raw_ids: set[str] = set() + census_resource_blocked_raw_ids: set[str] = set() + census_component_limit = ( + raw_artifact_limit if raw_artifact_limit is not None else RAW_MATERIALIZATION_CENSUS_COMPONENT_LIMIT + ) + if census_component_limit < 1: + raise ValueError("raw_artifact_limit must be positive") + census_components_attempted = 0 + if uncensused_raw_ids: + preliminary_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + for component in preliminary_components: + if not uncensused_raw_ids.intersection(component): + continue + if census_components_attempted >= census_component_limit: + break + census_components_attempted += 1 + seed = _raw_materialization_component_seed(candidates, component) + try: + census_historical_revision_evidence( + archive_root, + selected_raw_ids=[seed], + max_payload_bytes=RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES, + ) + except RawRevisionReplayResourceBlockedError as exc: + logger.warning( + "raw authority census resource-blocked for component containing %s: %s", + seed, + exc, + ) + census_resource_blocked_raw_ids.update(component) + except Exception: + logger.exception("raw authority census failed for component containing %s", seed) + census_failed_raw_ids.update(component) + candidates = _raw_materialization_candidate_ids( + config, + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + ) + relevant_raw_ids = list(candidates.expanded_raw_ids or tuple(candidates.raw_ids)) + uncensused_raw_ids = set(uncensused_historical_revision_raw_ids(archive_root, relevant_raw_ids)) + census_pending_raw_ids = tuple(sorted(uncensused_raw_ids | census_failed_raw_ids)) + if census_pending_raw_ids: + residual = _raw_authority_residual(candidates, census_pending_raw_ids=census_pending_raw_ids) + census_receipt = record_raw_authority_census( + archive_root, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="census", + quiescent=False, + scope=_raw_authority_scope( + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + raw_artifact_limit=raw_artifact_limit, + ), + residual=residual, + ) + metrics = _raw_materialization_base_metrics( + candidates, + recovered_census_count=recovered_census_count, + ) + metrics.update( + { + "raw_materialization_selected_count": 0.0, + "raw_materialization_selected_total_blob_bytes": 0.0, + "raw_materialization_selected_max_blob_bytes": 0.0, + "raw_materialization_executed_count": 0.0, + "raw_materialization_census_incomplete_raw_count": float(len(census_pending_raw_ids)), + "raw_materialization_census_component_limit": float(census_component_limit), + "raw_materialization_census_components_attempted": float(census_components_attempted), + "raw_materialization_census_sequence": float(census_receipt.sequence_no), + "raw_materialization_census_fixed_point": 0.0, + } + ) + if census_resource_blocked_raw_ids: + metrics["raw_materialization_resource_blocked_count"] = float(len(census_resource_blocked_raw_ids)) + metrics["raw_materialization_execute_blob_limit_bytes"] = float( + RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + ) + oversized = { + raw_id + for raw_id in census_resource_blocked_raw_ids + if _raw_materialization_component_blob_bytes(candidates, raw_id) + > RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + } + if oversized: + metrics["raw_materialization_oversized_count"] = float(len(oversized)) + stream_oversized = { + raw_id for raw_id in oversized if _raw_materialization_stream_safe(candidates, raw_id) + } + if stream_oversized: + metrics["raw_materialization_stream_oversized_count"] = float(len(stream_oversized)) + detail = ( + f"Raw replay planning paused until the persisted parser census completes for " + f"{len(census_pending_raw_ids):,} relevant raw(s)" + ) + if census_resource_blocked_raw_ids: + detail += ( + f"; {len(census_resource_blocked_raw_ids):,} raw(s) belong to authority components whose " + f"aggregate payload exceeds {_format_bytes(RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES)}" + ) + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=detail, + metrics=metrics, + census_receipt=census_receipt, + ) candidate_raw_ids = candidates.raw_ids - archive_root = _raw_materialization_archive_root(config) ordered_components = _raw_materialization_ordered_components(candidates, archive_root=archive_root) + plans = build_raw_replay_plans(archive_root, ordered_components) + plan_by_component = {plan.input_raw_ids: plan for plan in plans} all_blocked_components = [ component for component in ordered_components @@ -6407,7 +6685,7 @@ def repair_raw_materialization( blocked_component_raw_ids = {raw_id for component in blocked_components for raw_id in component} blocked_plan_outcomes = tuple( RawReplayPlanOutcome( - _raw_replay_plan_id(component), + plan_by_component[component].plan_id, component, RawReplayPlanStatus.RETRYABLE, "authority component exceeds the bounded replay resource envelope", @@ -6418,6 +6696,7 @@ def repair_raw_materialization( executable_components = [ component for component in selected_components if not blocked_component_raw_ids.intersection(component) ] + executable_plans = [plan_by_component[component] for component in executable_components] raw_ids = [_raw_materialization_component_seed(candidates, component) for component in executable_components] selected_component_raw_ids = {raw_id for component in selected_components for raw_id in component} selected_candidate_raw_ids = selected_component_raw_ids.intersection(candidate_raw_ids) @@ -6429,23 +6708,14 @@ def repair_raw_materialization( (_raw_materialization_component_blob_bytes(candidates, raw_id) for raw_id in selected_component_raw_ids), default=0, ) - metrics = { - "raw_materialization_candidate_count": float(len(candidate_raw_ids)), - "raw_materialization_selected_count": float(len(selected_candidate_raw_ids)), - "raw_materialization_missing_blob_count": float(missing_blobs), - "raw_materialization_missing_blob_source_available_count": float(candidates.missing_blob_source_available), - "raw_materialization_missing_blob_source_missing_count": float(candidates.missing_blob_source_missing), - "raw_materialization_already_parsed_count": float(candidates.already_parsed), - "raw_materialization_total_blob_bytes": float(candidates.total_blob_bytes), - "raw_materialization_max_blob_bytes": float(candidates.max_blob_bytes), - "raw_materialization_selected_total_blob_bytes": float(selected_total_bytes), - "raw_materialization_selected_max_blob_bytes": float(selected_max_bytes), - "raw_materialization_adoption_deferred_count": float(candidates.adoption_deferred), - "raw_materialization_authority_quarantined_count": float(candidates.authority_quarantined), - "raw_materialization_byte_authority_fragment_count": float(candidates.byte_authority_fragments), - "raw_materialization_byte_authority_quarantined_count": float(candidates.byte_authority_quarantined), - "raw_materialization_byte_authority_pending_count": float(candidates.byte_authority_pending), - } + metrics = _raw_materialization_base_metrics(candidates, recovered_census_count=recovered_census_count) + metrics.update( + { + "raw_materialization_selected_count": float(len(selected_candidate_raw_ids)), + "raw_materialization_selected_total_blob_bytes": float(selected_total_bytes), + "raw_materialization_selected_max_blob_bytes": float(selected_max_bytes), + } + ) if raw_artifact_limit is not None: metrics["raw_materialization_limit"] = float(raw_artifact_limit) metrics["raw_materialization_selected_component_count"] = float(len(selected_components)) @@ -6476,6 +6746,36 @@ def repair_raw_materialization( metrics["raw_materialization_execute_blob_limit_bytes"] = float(RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES) if oversized_stream_safe_raw_ids: metrics["raw_materialization_stream_oversized_count"] = float(len(oversized_stream_safe_raw_ids)) + selected_plan_ids = {plan_by_component[component].plan_id for component in selected_components} + executable_plan_ids = { + plan_by_component[component].plan_id + for component in ordered_components + if not all_blocked_component_raw_ids.intersection(component) + } + residual = _raw_authority_residual( + candidates, + resource_blocked_plan_ids=tuple( + sorted(plan_by_component[component].plan_id for component in all_blocked_components) + ), + ) + census_receipt = record_raw_authority_census( + archive_root, + plans, + selected_plan_ids=set() if dry_run else selected_plan_ids, + executable_plan_ids=executable_plan_ids, + mode="dry_run" if dry_run else "apply", + quiescent=True, + scope=_raw_authority_scope( + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + raw_artifact_limit=raw_artifact_limit, + ), + residual=residual, + ) + metrics["raw_materialization_census_sequence"] = float(census_receipt.sequence_no) + metrics["raw_materialization_census_fixed_point"] = float(census_receipt.fixed_point) if not candidate_raw_ids: detail = "Executable raw replay converged" if ( @@ -6515,12 +6815,13 @@ def repair_raw_materialization( ), detail=detail, metrics=metrics, + census_receipt=census_receipt, ) if dry_run: plan_outcomes = ( tuple( RawReplayPlanOutcome( - _raw_replay_plan_id(component), + plan_by_component[component].plan_id, component, RawReplayPlanStatus.RETRYABLE, "dry-run census selected this executable authority component", @@ -6554,19 +6855,81 @@ def repair_raw_materialization( detail=detail, metrics=metrics, plan_outcomes=plan_outcomes, + census_receipt=census_receipt, ) - from polylogue.sources.revision_backfill import ( - RawRevisionReplayResourceBlockedError, - RevisionBackfillResult, - backfill_historical_revision_evidence, - ) + for outcome in blocked_plan_outcomes: + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + + stale_outcomes: list[RawReplayPlanOutcome] = [] + validated_plans: list[RawReplayPlan] = [] + for plan in executable_plans: + valid, observed = validate_raw_replay_plan(archive_root, plan) + if valid: + validated_plans.append(plan) + else: + stale_outcomes.append(reject_stale_raw_replay_plan(archive_root, census_receipt.census_id, plan, observed)) + if stale_outcomes: + carried = [ + RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.CARRIED_FORWARD, + "another selected plan failed immutable precondition validation", + "resolve the durable stale-plan blocker before retrying this unchanged plan", + ) + for plan in validated_plans + ] + for outcome in carried: + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + stale_candidates = _raw_materialization_candidate_ids( + config, + raw_artifact_id=raw_artifact_id, + provider=provider, + source_family=source_family, + source_root=source_root, + ) + stale_post_plans, stale_post_residual = _raw_authority_postflight_snapshot( + archive_root, + stale_candidates, + ) + census_receipt = finalize_raw_authority_census( + archive_root, + census_receipt.census_id, + post_plans=stale_post_plans, + post_residual=stale_post_residual, + ) + plan_outcomes = tuple(stale_outcomes + carried) + blocked_plan_outcomes + plan_count, carried_forward_count, conservation_error_count = _raw_replay_conservation_metrics( + plans, + selected_plan_ids, + plan_outcomes, + ) + metrics["raw_materialization_plan_rejected_stale_count"] = float(len(stale_outcomes)) + metrics["raw_materialization_plan_carried_forward_count"] = float(carried_forward_count) + metrics["raw_materialization_plan_outcome_count"] = float(plan_count) + metrics["raw_materialization_plan_conservation_error_count"] = float(conservation_error_count) + return _internal_derived_repair_result( + "raw_materialization", + repaired_count=0, + success=False, + detail=( + f"Rejected {len(stale_outcomes):,} stale immutable replay plan(s); " + "automatic convergence is now fail-closed behind durable blocker evidence" + ), + metrics=metrics, + plan_outcomes=plan_outcomes, + census_receipt=census_receipt, + ) + + from polylogue.sources.revision_backfill import RevisionBackfillResult, backfill_historical_revision_evidence executable_raw_ids = raw_ids metrics["raw_materialization_executed_count"] = float(len(executable_raw_ids)) replay_parts: list[RevisionBackfillResult] = [] execution_outcomes: list[RawReplayPlanOutcome] = [] - for component, raw_id in zip(executable_components, executable_raw_ids, strict=True): + for plan, raw_id in zip(executable_plans, executable_raw_ids, strict=True): + component = plan.input_raw_ids try: part = backfill_historical_revision_evidence( archive_root, @@ -6577,27 +6940,54 @@ def repair_raw_materialization( metrics["raw_materialization_resource_blocked_count"] = max( metrics.get("raw_materialization_resource_blocked_count", 0.0), float(len(exc.raw_ids)) ) - execution_outcomes.append( - RawReplayPlanOutcome( - _raw_replay_plan_id(component), - component, - RawReplayPlanStatus.RETRYABLE, - "expanded authority component exceeded the bounded replay resource envelope", - "retry the same plan after streaming/resource admission is available", - ) + outcome = RawReplayPlanOutcome( + plan.plan_id, + component, + RawReplayPlanStatus.RETRYABLE, + "expanded authority component exceeded the bounded replay resource envelope", + "retry the same plan after streaming/resource admission is available", ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + execution_outcomes.append(outcome) continue except Exception as exc: - logger.exception("raw replay plan %s failed", _raw_replay_plan_id(component)) - execution_outcomes.append( - RawReplayPlanOutcome( - _raw_replay_plan_id(component), + logger.exception("raw replay plan %s failed", plan.plan_id) + application_receipt = raw_replay_application_receipt(archive_root, plan) + receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) + if receipt_valid: + outcome = RawReplayPlanOutcome( + plan.plan_id, component, - RawReplayPlanStatus.RETRYABLE, - f"component execution raised {type(exc).__name__}: {exc}", - "retry this plan after independent components have received a turn", + RawReplayPlanStatus.EXECUTED, + f"component reached exact durable postconditions before {type(exc).__name__}", + "none", + application_receipt, ) - ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + else: + plan_still_valid, _ = validate_raw_replay_plan(archive_root, plan) + if plan_still_valid: + outcome = RawReplayPlanOutcome( + plan.plan_id, + component, + RawReplayPlanStatus.RETRYABLE, + f"component execution raised {type(exc).__name__}: {exc}", + "retry this unchanged plan after independent components have received a turn", + application_receipt, + ) + record_raw_replay_outcome(archive_root, census_receipt.census_id, outcome) + else: + outcome = reject_invalid_raw_replay_application( + archive_root, + census_receipt.census_id, + plan, + application_receipt, + ( + f"component execution raised {type(exc).__name__}: {exc}", + *receipt_problems, + ), + ) + execution_outcomes.append(outcome) continue replay_parts.append(part) current = _raw_materialization_candidate_ids( @@ -6607,7 +6997,33 @@ def repair_raw_materialization( source_family=source_family, source_root=source_root, ) - execution_outcomes.extend(_raw_replay_plan_outcomes(archive_root, [component], remaining=current)) + component_outcomes = _raw_replay_plan_outcomes(archive_root, [plan], remaining=current) + for outcome in component_outcomes: + application_receipt = raw_replay_application_receipt(archive_root, plan) + receipted = dataclasses.replace(outcome, application_receipt=application_receipt) + if outcome.status is RawReplayPlanStatus.EXECUTED: + receipt_valid, receipt_problems = validate_raw_replay_application_receipt(plan, application_receipt) + if not receipt_valid: + receipted = reject_invalid_raw_replay_application( + archive_root, + census_receipt.census_id, + plan, + application_receipt, + receipt_problems, + ) + else: + record_raw_replay_outcome(archive_root, census_receipt.census_id, receipted) + elif outcome.status is RawReplayPlanStatus.REJECTED_STALE: + receipted = reject_invalid_raw_replay_application( + archive_root, + census_receipt.census_id, + plan, + application_receipt, + (outcome.reason,), + ) + else: + record_raw_replay_outcome(archive_root, census_receipt.census_id, receipted) + execution_outcomes.append(receipted) replay = RevisionBackfillResult( scanned=sum(part.scanned for part in replay_parts), @@ -6640,14 +7056,25 @@ def repair_raw_materialization( } ) plan_outcomes = tuple(execution_outcomes) + blocked_plan_outcomes + post_plans, post_residual = _raw_authority_postflight_snapshot(archive_root, remaining) + census_receipt = finalize_raw_authority_census( + archive_root, + census_receipt.census_id, + post_plans=post_plans, + post_residual=post_residual, + ) for status in RawReplayPlanStatus: metrics[f"raw_materialization_plan_{status.value}_count"] = float( sum(outcome.status is status for outcome in plan_outcomes) ) - metrics["raw_materialization_plan_outcome_count"] = float(len(plan_outcomes)) - metrics["raw_materialization_plan_conservation_error_count"] = float( - sum(outcome.status is RawReplayPlanStatus.REJECTED_STALE for outcome in plan_outcomes) + plan_count, carried_forward_count, conservation_error_count = _raw_replay_conservation_metrics( + plans, + selected_plan_ids, + plan_outcomes, ) + metrics["raw_materialization_plan_carried_forward_count"] = float(carried_forward_count) + metrics["raw_materialization_plan_outcome_count"] = float(plan_count) + metrics["raw_materialization_plan_conservation_error_count"] = float(conservation_error_count) success = ( not remaining.raw_ids and remaining.missing_blobs == 0 @@ -6713,6 +7140,7 @@ def repair_raw_materialization( detail=detail, metrics=metrics, plan_outcomes=plan_outcomes, + census_receipt=census_receipt, ) diff --git a/polylogue/storage/sqlite/archive_tiers/source.py b/polylogue/storage/sqlite/archive_tiers/source.py index eb94034ed5..037161676b 100644 --- a/polylogue/storage/sqlite/archive_tiers/source.py +++ b/polylogue/storage/sqlite/archive_tiers/source.py @@ -9,7 +9,7 @@ from polylogue.core.enums import ArtifactSupportStatus, Origin, Provider, ValidationMode, ValidationStatus from polylogue.storage.sqlite.archive_tiers.common import check, nullable_check -SOURCE_SCHEMA_VERSION = 12 +SOURCE_SCHEMA_VERSION = 13 SOURCE_DDL = f""" CREATE TABLE IF NOT EXISTS raw_sessions ( @@ -101,6 +101,120 @@ detail TEXT NOT NULL DEFAULT '' ) STRICT; +-- Durable authority reconciliation ledger. The source tier owns this +-- evidence because index.db and ops.db are rebuildable/disposable: neither +-- can be the authority for whether an accepted replay plan was conserved. +CREATE TABLE IF NOT EXISTS raw_authority_parser_census ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + parser_fingerprint TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('complete', 'failed')), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + detail TEXT NOT NULL DEFAULT '', + censused_at_ms INTEGER NOT NULL CHECK(censused_at_ms >= 0) +) STRICT; + +CREATE TABLE IF NOT EXISTS raw_authority_censuses ( + census_id TEXT PRIMARY KEY, + sequence_no INTEGER NOT NULL UNIQUE CHECK(sequence_no > 0), + scope_json TEXT NOT NULL CHECK(json_valid(scope_json)), + residual_json TEXT NOT NULL CHECK(json_valid(residual_json)), + parser_fingerprint TEXT NOT NULL, + mode TEXT NOT NULL CHECK(mode IN ('census', 'dry_run', 'apply')), + lifecycle_status TEXT NOT NULL CHECK(lifecycle_status IN ('planned', 'completed', 'interrupted')), + quiescent INTEGER NOT NULL CHECK(quiescent IN (0, 1)), + inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), + residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), + plan_count INTEGER NOT NULL CHECK(plan_count >= 0), + post_inventory_digest TEXT CHECK(post_inventory_digest IS NULL OR length(post_inventory_digest) = 64), + post_residual_json TEXT CHECK(post_residual_json IS NULL OR json_valid(post_residual_json)), + post_residual_digest TEXT CHECK(post_residual_digest IS NULL OR length(post_residual_digest) = 64), + post_plan_count INTEGER CHECK(post_plan_count IS NULL OR post_plan_count >= 0), + postflight_at_ms INTEGER CHECK(postflight_at_ms IS NULL OR postflight_at_ms >= created_at_ms), + executable_plan_count INTEGER NOT NULL CHECK(executable_plan_count >= 0), + residual_plan_count INTEGER NOT NULL CHECK(residual_plan_count >= 0), + predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), + fixed_point INTEGER NOT NULL DEFAULT 0 CHECK(fixed_point IN (0, 1)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + completed_at_ms INTEGER CHECK(completed_at_ms IS NULL OR completed_at_ms >= created_at_ms), + CHECK(plan_count >= executable_plan_count), + CHECK(plan_count >= residual_plan_count), + CHECK(plan_count = executable_plan_count + residual_plan_count), + CHECK( + (lifecycle_status = 'planned' AND completed_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) + ), + CHECK( + (lifecycle_status = 'planned' AND post_inventory_digest IS NULL + AND post_residual_json IS NULL AND post_residual_digest IS NULL + AND post_plan_count IS NULL AND postflight_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') + AND post_inventory_digest IS NOT NULL AND post_residual_json IS NOT NULL + AND post_residual_digest IS NOT NULL AND post_plan_count IS NOT NULL + AND postflight_at_ms IS NOT NULL) + ) +) STRICT; + +CREATE TABLE IF NOT EXISTS raw_authority_plans ( + plan_id TEXT PRIMARY KEY, + input_digest TEXT NOT NULL CHECK(length(input_digest) = 64), + input_raw_ids_json TEXT NOT NULL CHECK(json_valid(input_raw_ids_json)), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + authority_witness_json TEXT NOT NULL CHECK(json_valid(authority_witness_json)), + source_preconditions_json TEXT NOT NULL CHECK(json_valid(source_preconditions_json)), + index_preconditions_json TEXT NOT NULL CHECK(json_valid(index_preconditions_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) +) STRICT; + +CREATE TABLE IF NOT EXISTS raw_authority_census_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + selected INTEGER NOT NULL CHECK(selected IN (0, 1)), + outcome_status TEXT NOT NULL CHECK(outcome_status IN ( + 'executed', 'retryable', 'deferred', 'terminal', + 'rejected_stale', 'carried_forward' + )), + reason TEXT NOT NULL, + next_action TEXT NOT NULL, + application_receipt_json TEXT NOT NULL DEFAULT '{{}}' CHECK(json_valid(application_receipt_json)), + outcome_recorded INTEGER NOT NULL DEFAULT 0 CHECK(outcome_recorded IN (0, 1)), + recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_raw_authority_census_plans_status +ON raw_authority_census_plans(census_id, outcome_status, ordinal); + +CREATE INDEX IF NOT EXISTS idx_raw_authority_census_plans_attempts +ON raw_authority_census_plans(plan_id, recorded_at_ms DESC) +WHERE selected = 1; + +CREATE TABLE IF NOT EXISTS raw_authority_census_post_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + +CREATE TABLE IF NOT EXISTS raw_authority_blockers ( + blocker_id TEXT PRIMARY KEY, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id), + reason TEXT NOT NULL, + expected_json TEXT NOT NULL CHECK(json_valid(expected_json)), + observed_json TEXT NOT NULL CHECK(json_valid(observed_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + resolved_at_ms INTEGER CHECK(resolved_at_ms IS NULL OR resolved_at_ms >= created_at_ms), + resolution TEXT, + CHECK((resolved_at_ms IS NULL) = (resolution IS NULL)) +) STRICT; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_authority_blockers_open_plan +ON raw_authority_blockers(plan_id) +WHERE resolved_at_ms IS NULL; + CREATE TABLE IF NOT EXISTS blob_refs ( blob_hash BLOB NOT NULL CHECK(length(blob_hash) = 32), ref_id TEXT NOT NULL, diff --git a/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql new file mode 100644 index 0000000000..a5ec20a78b --- /dev/null +++ b/polylogue/storage/sqlite/migrations/source/013_raw_authority_ledger.sql @@ -0,0 +1,111 @@ +-- Durable, restart-safe conservation ledger for raw authority reconciliation. +CREATE TABLE raw_authority_parser_census ( + raw_id TEXT PRIMARY KEY REFERENCES raw_sessions(raw_id) ON DELETE CASCADE, + parser_fingerprint TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('complete', 'failed')), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + detail TEXT NOT NULL DEFAULT '', + censused_at_ms INTEGER NOT NULL CHECK(censused_at_ms >= 0) +) STRICT; + +CREATE TABLE raw_authority_censuses ( + census_id TEXT PRIMARY KEY, + sequence_no INTEGER NOT NULL UNIQUE CHECK(sequence_no > 0), + scope_json TEXT NOT NULL CHECK(json_valid(scope_json)), + residual_json TEXT NOT NULL CHECK(json_valid(residual_json)), + parser_fingerprint TEXT NOT NULL, + mode TEXT NOT NULL CHECK(mode IN ('census', 'dry_run', 'apply')), + lifecycle_status TEXT NOT NULL CHECK(lifecycle_status IN ('planned', 'completed', 'interrupted')), + quiescent INTEGER NOT NULL CHECK(quiescent IN (0, 1)), + inventory_digest TEXT NOT NULL CHECK(length(inventory_digest) = 64), + residual_digest TEXT NOT NULL CHECK(length(residual_digest) = 64), + plan_count INTEGER NOT NULL CHECK(plan_count >= 0), + post_inventory_digest TEXT CHECK(post_inventory_digest IS NULL OR length(post_inventory_digest) = 64), + post_residual_json TEXT CHECK(post_residual_json IS NULL OR json_valid(post_residual_json)), + post_residual_digest TEXT CHECK(post_residual_digest IS NULL OR length(post_residual_digest) = 64), + post_plan_count INTEGER CHECK(post_plan_count IS NULL OR post_plan_count >= 0), + postflight_at_ms INTEGER CHECK(postflight_at_ms IS NULL OR postflight_at_ms >= created_at_ms), + executable_plan_count INTEGER NOT NULL CHECK(executable_plan_count >= 0), + residual_plan_count INTEGER NOT NULL CHECK(residual_plan_count >= 0), + predecessor_census_id TEXT REFERENCES raw_authority_censuses(census_id), + fixed_point INTEGER NOT NULL DEFAULT 0 CHECK(fixed_point IN (0, 1)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + completed_at_ms INTEGER CHECK(completed_at_ms IS NULL OR completed_at_ms >= created_at_ms), + CHECK(plan_count >= executable_plan_count), + CHECK(plan_count >= residual_plan_count), + CHECK(plan_count = executable_plan_count + residual_plan_count), + CHECK( + (lifecycle_status = 'planned' AND completed_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') AND completed_at_ms IS NOT NULL) + ), + CHECK( + (lifecycle_status = 'planned' AND post_inventory_digest IS NULL + AND post_residual_json IS NULL AND post_residual_digest IS NULL + AND post_plan_count IS NULL AND postflight_at_ms IS NULL) + OR (lifecycle_status IN ('completed', 'interrupted') + AND post_inventory_digest IS NOT NULL AND post_residual_json IS NOT NULL + AND post_residual_digest IS NOT NULL AND post_plan_count IS NOT NULL + AND postflight_at_ms IS NOT NULL) + ) +) STRICT; + +CREATE TABLE raw_authority_plans ( + plan_id TEXT PRIMARY KEY, + input_digest TEXT NOT NULL CHECK(length(input_digest) = 64), + input_raw_ids_json TEXT NOT NULL CHECK(json_valid(input_raw_ids_json)), + logical_keys_json TEXT NOT NULL CHECK(json_valid(logical_keys_json)), + authority_witness_json TEXT NOT NULL CHECK(json_valid(authority_witness_json)), + source_preconditions_json TEXT NOT NULL CHECK(json_valid(source_preconditions_json)), + index_preconditions_json TEXT NOT NULL CHECK(json_valid(index_preconditions_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) +) STRICT; + +CREATE TABLE raw_authority_census_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + selected INTEGER NOT NULL CHECK(selected IN (0, 1)), + outcome_status TEXT NOT NULL CHECK(outcome_status IN ( + 'executed', 'retryable', 'deferred', 'terminal', + 'rejected_stale', 'carried_forward' + )), + reason TEXT NOT NULL, + next_action TEXT NOT NULL, + application_receipt_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(application_receipt_json)), + outcome_recorded INTEGER NOT NULL DEFAULT 0 CHECK(outcome_recorded IN (0, 1)), + recorded_at_ms INTEGER NOT NULL CHECK(recorded_at_ms >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + +CREATE INDEX idx_raw_authority_census_plans_status +ON raw_authority_census_plans(census_id, outcome_status, ordinal); + +CREATE INDEX idx_raw_authority_census_plans_attempts +ON raw_authority_census_plans(plan_id, recorded_at_ms DESC) +WHERE selected = 1; + +CREATE TABLE raw_authority_census_post_plans ( + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id) ON DELETE CASCADE, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + ordinal INTEGER NOT NULL CHECK(ordinal >= 0), + PRIMARY KEY(census_id, plan_id), + UNIQUE(census_id, ordinal) +) STRICT; + +CREATE TABLE raw_authority_blockers ( + blocker_id TEXT PRIMARY KEY, + plan_id TEXT NOT NULL REFERENCES raw_authority_plans(plan_id), + census_id TEXT NOT NULL REFERENCES raw_authority_censuses(census_id), + reason TEXT NOT NULL, + expected_json TEXT NOT NULL CHECK(json_valid(expected_json)), + observed_json TEXT NOT NULL CHECK(json_valid(observed_json)), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + resolved_at_ms INTEGER CHECK(resolved_at_ms IS NULL OR resolved_at_ms >= created_at_ms), + resolution TEXT, + CHECK((resolved_at_ms IS NULL) = (resolution IS NULL)) +) STRICT; + +CREATE UNIQUE INDEX idx_raw_authority_blockers_open_plan +ON raw_authority_blockers(plan_id) +WHERE resolved_at_ms IS NULL; diff --git a/tests/infra/mcp.py b/tests/infra/mcp.py index 223a8f3287..258ab5e84f 100644 --- a/tests/infra/mcp.py +++ b/tests/infra/mcp.py @@ -128,6 +128,8 @@ EXPECTED_RESOURCE_TEMPLATE_URIS = { "polylogue://session/{conv_id}", + "polylogue://raw-authority-census/{census_id}/{offset}", + "polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}", } EXPECTED_PROMPT_NAMES = { diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index 642ac09d9a..3ecf65a687 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -13,10 +13,12 @@ from polylogue.cli.commands.maintenance import _rebuild_index as maintenance_rebuild_index from polylogue.config import Config from polylogue.core.enums import Provider +from polylogue.core.json import json_document from polylogue.maintenance.replay import rebuild_index_from_source from polylogue.sources.live.cursor import CursorStore from polylogue.storage.blob_gc import read_gc_history from polylogue.storage.blob_publication import ArchiveBlobPublisher +from polylogue.storage.raw_authority import RawReplayPlan, record_raw_authority_census from polylogue.storage.sqlite.archive_tiers.archive import ArchiveSessionSearchHit, ArchiveSessionSummary from polylogue.storage.sqlite.archive_tiers.archive_init import ( ArchiveInitResult, @@ -32,6 +34,142 @@ _ARCHIVE_TIERS = ("source.db", "index.db", "embeddings.db", "ops.db", "user.db") +def test_raw_authority_census_cli_resolves_receipt_handle( + cli_workspace: dict[str, Path], + cli_runner: CliRunner, +) -> None: + root = cli_workspace["archive_root"] + receipt = record_raw_authority_census( + root, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"source_family": "codex"}, + residual={}, + ) + + result = cli_runner.invoke( + cli, + [ + "--plain", + "ops", + "maintenance", + "raw-authority-census", + receipt.query_handle, + "--output-format", + "json", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["query_handle"] == receipt.query_handle + assert payload["census"]["census_id"] == receipt.census_id + + +def test_raw_authority_cli_bounds_oversized_plan_and_resolves_detail( + cli_workspace: dict[str, Path], + cli_runner: CliRunner, +) -> None: + root = cli_workspace["archive_root"] + raw_ids = tuple(f"raw-{index:05d}" for index in range(2_000)) + plan = RawReplayPlan( + "raw-replay:cli-oversized", + "b" * 64, + raw_ids, + ("codex:oversized",), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + ) + receipt = record_raw_authority_census( + root, + (plan,), + selected_plan_ids=set(), + executable_plan_ids={plan.plan_id}, + mode="dry_run", + quiescent=True, + scope={"test": "cli-oversized"}, + residual={}, + ) + + census_result = cli_runner.invoke( + cli, + [ + "--plain", + "ops", + "maintenance", + "raw-authority-census", + receipt.query_handle, + "--limit", + "1", + "--output-format", + "json", + ], + catch_exceptions=False, + ) + assert census_result.exit_code == 0 + assert len(census_result.output) < 8_000 + census_payload = json.loads(census_result.output) + item = census_payload["plans"][0] + assert item["plan"]["input_raw_count"] == 2_000 + assert "raw-01999" not in census_result.output + + detail_result = cli_runner.invoke( + cli, + [ + "--plain", + "ops", + "maintenance", + "raw-authority-detail", + item["detail_query_handle"], + "--chunk-chars", + "256", + "--output-format", + "json", + ], + catch_exceptions=False, + ) + assert detail_result.exit_code == 0 + detail_payload = json.loads(detail_result.output) + assert len(detail_payload["chunk"]) <= 256 + assert detail_payload["next_query_handle"] is not None + + +def test_raw_authority_blocker_resolution_cli_requires_confirmation( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + def resolve(_root: Path, blocker_id: str, *, resolution: str) -> dict[str, object]: + calls.append((blocker_id, resolution)) + return {"blocker_id": blocker_id, "current_plan": {"plan_id": "current-plan"}} + + monkeypatch.setattr("polylogue.storage.raw_authority.resolve_raw_authority_blocker", resolve) + base = [ + "--plain", + "ops", + "maintenance", + "raw-authority-blocker-resolve", + "--blocker-id", + "blocker-1", + "--reason", + "reviewed current evidence", + ] + refused = cli_runner.invoke(cli, base) + accepted = cli_runner.invoke(cli, [*base, "--yes"], catch_exceptions=False) + + assert refused.exit_code != 0 + assert "without --yes" in refused.output + assert accepted.exit_code == 0 + assert "Resolved blocker-1" in accepted.output + assert calls == [("blocker-1", "reviewed current evidence")] + + def _stage_uninitialized_archive(cli_workspace: dict[str, Path]) -> None: """Reset the workspace to an uninitialized state for plan/init tests. diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index c6a902de57..5f9944ab51 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -21,6 +21,7 @@ from polylogue.daemon.health import DaemonHealth, HealthSeverity, HealthTier from polylogue.sources.live import WatchSource from polylogue.sources.live.cursor import CursorStore +from polylogue.storage.raw_authority import RawReplayPlanOutcome, RawReplayPlanStatus from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database from polylogue.storage.sqlite.archive_tiers.embeddings import EMBEDDINGS_SCHEMA_VERSION from polylogue.storage.sqlite.archive_tiers.index import INDEX_SCHEMA_VERSION @@ -571,17 +572,12 @@ def test_raw_materialization_pass_emits_conserved_plan_receipt(monkeypatch: pyte from polylogue.daemon import cli as daemon_cli events: list[tuple[str, dict[str, object]]] = [] - outcome = SimpleNamespace( - status=SimpleNamespace(value="executed"), + outcome = RawReplayPlanOutcome( plan_id="raw-replay:stable", + input_raw_ids=("raw-a",), + status=RawReplayPlanStatus.EXECUTED, reason="applied", - to_dict=lambda: { - "plan_id": "raw-replay:stable", - "input_raw_ids": ["raw-a"], - "status": "executed", - "reason": "applied", - "next_action": "none", - }, + next_action="none", ) monkeypatch.setattr( "polylogue.daemon.events.emit_daemon_event", @@ -614,7 +610,9 @@ def test_raw_materialization_pass_emits_conserved_plan_receipt(monkeypatch: pyte "raw_materialization_candidate_count": 1.0, "raw_materialization_remaining_candidate_count": 0.0, }, - "plan_outcomes": [outcome.to_dict()], + "plan_outcome_count": 1, + "plan_outcome_sample": [outcome.to_summary_dict()], + "plan_outcome_sample_truncated": False, }, ) ] @@ -641,7 +639,102 @@ def test_raw_materialization_pass_emits_zero_work_receipt(monkeypatch: pytest.Mo ) assert events[0][1]["success"] is True - assert events[0][1]["plan_outcomes"] == [] + assert events[0][1]["plan_outcome_count"] == 0 + assert events[0][1]["plan_outcome_sample"] == [] + assert events[0][1]["plan_outcome_sample_truncated"] is False + + +def test_raw_materialization_pass_bounds_outcome_sample(monkeypatch: pytest.MonkeyPatch) -> None: + from polylogue.daemon import cli as daemon_cli + + events: list[tuple[str, dict[str, object]]] = [] + outcomes = tuple( + RawReplayPlanOutcome( + plan_id=f"plan:{index}", + input_raw_ids=(f"raw:{index}",), + status=RawReplayPlanStatus.CARRIED_FORWARD, + reason="not selected in bounded pass", + next_action="retry later", + ) + for index in range(9) + ) + monkeypatch.setattr( + "polylogue.daemon.events.emit_daemon_event", + lambda kind, *, payload: events.append((kind, payload)), + ) + + daemon_cli._emit_raw_materialization_pass( + SimpleNamespace( + success=True, + repaired_count=0, + detail="bounded", + metrics={}, + plan_outcomes=outcomes, + ) + ) + + assert events[0][1]["plan_outcome_count"] == 9 + assert len(cast(list[object], events[0][1]["plan_outcome_sample"])) == 8 + assert events[0][1]["plan_outcome_sample_truncated"] is True + + +def test_raw_materialization_pass_projects_durable_census_handle(monkeypatch: pytest.MonkeyPatch) -> None: + from polylogue.daemon import cli as daemon_cli + + events: list[tuple[str, dict[str, object]]] = [] + monkeypatch.setattr( + "polylogue.daemon.events.emit_daemon_event", + lambda kind, *, payload: events.append((kind, payload)), + ) + monkeypatch.setattr(os, "urandom", lambda _size: b"c" * 16) + census = SimpleNamespace( + census_id="census:2:inventory:residual", + sequence_no=2, + inventory_digest="a" * 64, + residual_digest="b" * 64, + plan_count=3, + post_inventory_digest="c" * 64, + post_residual_digest="d" * 64, + post_plan_count=2, + executable_plan_count=1, + residual_plan_count=2, + predecessor_census_id="census:1:inventory:residual", + mode="apply", + lifecycle_status="completed", + quiescent=True, + fixed_point=False, + query_handle="polylogue://raw-authority-census/census:2:inventory:residual/0", + ) + + daemon_cli._emit_raw_materialization_pass( + SimpleNamespace( + success=False, + repaired_count=0, + detail="bounded", + metrics={}, + plan_outcomes=(), + census_receipt=census, + ) + ) + + assert events[0][1]["census"] == { + "census_id": census.census_id, + "sequence_no": 2, + "inventory_digest": "a" * 64, + "residual_digest": "b" * 64, + "plan_count": 3, + "post_inventory_digest": "c" * 64, + "post_residual_digest": "d" * 64, + "post_plan_count": 2, + "executable_plan_count": 1, + "residual_plan_count": 2, + "predecessor_census_id": census.predecessor_census_id, + "mode": "apply", + "lifecycle_status": "completed", + "quiescent": True, + "fixed_point": False, + "query_handle": census.query_handle, + } def test_raw_materialization_closes_fts_on_cancellation( diff --git a/tests/unit/mcp/test_server_surfaces.py b/tests/unit/mcp/test_server_surfaces.py index 4c5319a800..58be4a5519 100644 --- a/tests/unit/mcp/test_server_surfaces.py +++ b/tests/unit/mcp/test_server_surfaces.py @@ -17,9 +17,11 @@ from polylogue.archive.models import Session, SessionSummary from polylogue.archive.semantic.content_projection import ContentProjectionSpec from polylogue.core.enums import AssertionKind, AssertionStatus, AssertionVisibility, BlockType, BranchType, Provider +from polylogue.core.json import json_document from polylogue.core.refs import EvidenceRef from polylogue.core.types import SessionId from polylogue.sources.parsers.base import ParsedContentBlock, ParsedMessage, ParsedSession +from polylogue.storage.raw_authority import RawReplayPlan, record_raw_authority_census from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.user_write import ArchiveAssertionEnvelope from polylogue.surfaces.payloads import ( @@ -558,6 +560,92 @@ def test_readiness_resource(self: object, mcp_server: MCPServerUnderTest) -> Non assert component_readiness["index"]["state"] == "degraded" assert component_readiness["index"]["caveats"] == ["messages_fts_row_mismatch"] + def test_raw_authority_census_query_handle_resolves_bounded_ledger( + self: object, mcp_server: MCPServerUnderTest, tmp_path: Path + ) -> None: + archive_root = tmp_path / "archive" + with ArchiveStore(archive_root): + pass + receipt = record_raw_authority_census( + archive_root, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"origin": "codex-session"}, + residual={}, + ) + with patch("polylogue.mcp.server._get_config") as mock_get_config: + mock_get_config.return_value = SimpleNamespace( + archive_root=archive_root, + db_path=archive_root / "index.db", + ) + result = invoke_surface( + mcp_server._resource_manager._templates["polylogue://raw-authority-census/{census_id}/{offset}"].fn, + census_id=receipt.census_id, + offset="0", + ) + + payload = json.loads(result) + assert payload["query_handle"] == receipt.query_handle + assert payload["census"]["census_id"] == receipt.census_id + assert payload["plans"] == [] + + def test_raw_authority_mcp_bounds_oversized_plan_and_pages_detail( + self: object, mcp_server: MCPServerUnderTest, tmp_path: Path + ) -> None: + archive_root = tmp_path / "archive" + with ArchiveStore(archive_root): + pass + raw_ids = tuple(f"raw-{index:05d}" for index in range(2_000)) + plan = RawReplayPlan( + "raw-replay:mcp-oversized", + "c" * 64, + raw_ids, + ("codex:oversized",), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + json_document({"raw_ids": list(raw_ids)}), + ) + receipt = record_raw_authority_census( + archive_root, + (plan,), + selected_plan_ids=set(), + executable_plan_ids={plan.plan_id}, + mode="dry_run", + quiescent=True, + scope={"test": "mcp-oversized"}, + residual={}, + ) + with patch("polylogue.mcp.server._get_config") as mock_get_config: + mock_get_config.return_value = SimpleNamespace( + archive_root=archive_root, + db_path=archive_root / "index.db", + ) + census_result = invoke_surface( + mcp_server._resource_manager._templates["polylogue://raw-authority-census/{census_id}/{offset}"].fn, + census_id=receipt.census_id, + offset="0", + ) + census_payload = json.loads(census_result) + item = census_payload["plans"][0] + assert len(census_result) < 8_000 + assert item["plan"]["input_raw_count"] == 2_000 + detail_result = invoke_surface( + mcp_server._resource_manager._templates[ + "polylogue://raw-authority-detail/{census_id}/{record_id}/{revision}/{offset}" + ].fn, + census_id=receipt.census_id, + record_id=plan.plan_id, + revision="current", + offset="0", + ) + + detail_payload = json.loads(detail_result) + assert len(detail_payload["chunk"]) <= 16_384 + assert detail_payload["next_query_handle"] is not None + class TestArchiveGenericToolSurfaces: @pytest.mark.asyncio diff --git a/tests/unit/sources/test_revision_backfill.py b/tests/unit/sources/test_revision_backfill.py index 59ea6e21d5..f8036a9497 100644 --- a/tests/unit/sources/test_revision_backfill.py +++ b/tests/unit/sources/test_revision_backfill.py @@ -106,6 +106,16 @@ def test_historical_backfill_selects_prefix_newest_independent_of_acquisition_or assert result.classified_full == 2 assert result.replayed_logical_sources == 1 assert result.quarantined == 1 + with sqlite3.connect(tmp_path / "source.db") as conn: + parser_census = conn.execute( + """ + SELECT status, COUNT(*) + FROM raw_authority_parser_census + WHERE parser_fingerprint = 'revision-membership-v1' + GROUP BY status ORDER BY status + """ + ).fetchall() + assert parser_census == [("complete", 2), ("failed", 1)] with sqlite3.connect(tmp_path / "index.db") as conn: assert conn.execute("SELECT message_count, raw_id FROM sessions").fetchone() == (2, newest_raw_id) diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index 4527a35734..850d060b31 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -489,7 +489,7 @@ def test_source_tier_v1_migrates_to_current_without_native_uniqueness( result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) assert result.from_version == 1 assert result.to_version == SOURCE_SCHEMA_VERSION - assert result.applied_versions == (2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) + assert result.applied_versions == (2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13) assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == SOURCE_SCHEMA_VERSION columns = {str(row[1]) for row in conn.execute("PRAGMA table_info('raw_sessions')")} assert "predecessor_source_revision" in columns @@ -553,6 +553,12 @@ def test_source_publication_backfill_requires_verified_backup( ) -> None: db_path = workspace_env["archive_root"] / "source.db" with sqlite3.connect(db_path) as conn: + conn.execute("DROP TABLE raw_authority_parser_census") + conn.execute("DROP TABLE raw_authority_blockers") + conn.execute("DROP TABLE raw_authority_census_post_plans") + conn.execute("DROP TABLE raw_authority_census_plans") + conn.execute("DROP TABLE raw_authority_plans") + conn.execute("DROP TABLE raw_authority_censuses") conn.execute("DROP TABLE excised_content") conn.execute("DROP TABLE sinex_publication_segments") conn.execute("DROP TABLE sinex_publication_receipts") @@ -569,8 +575,8 @@ def test_source_publication_backfill_requires_verified_backup( result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) assert result.from_version == 9 - assert result.to_version == SOURCE_SCHEMA_VERSION == 12 - assert result.applied_versions == (10, 11, 12) + assert result.to_version == SOURCE_SCHEMA_VERSION == 13 + assert result.applied_versions == (10, 11, 12, 13) assert result.backup_receipt == manifest.with_name("verification-receipt.json") tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} assert { @@ -579,6 +585,12 @@ def test_source_publication_backfill_requires_verified_backup( "sinex_publication_segments", "sinex_publication_receipts", "excised_content", + "raw_authority_censuses", + "raw_authority_parser_census", + "raw_authority_plans", + "raw_authority_census_plans", + "raw_authority_census_post_plans", + "raw_authority_blockers", } <= tables @@ -643,6 +655,9 @@ def test_source_tier_v7_expands_origin_checks_with_verified_backup( " CHECK(revision_authority IN ('asserted', 'byte_proven', 'quarantined'))\n" " ,predecessor_source_revision TEXT\n", ) + authority_start = old_ddl.index("-- Durable authority reconciliation ledger.") + authority_end = old_ddl.index("CREATE TABLE IF NOT EXISTS blob_refs", authority_start) + old_ddl = old_ddl[:authority_start] + old_ddl[authority_end:] # Migration 010 adds `excised_content` (polylogue-27m) -- a v7 snapshot # predates it, same as it predates the beads-origin/capture_mode diffs # stripped above. Without this, the fixture (built from the CURRENT @@ -715,8 +730,8 @@ def test_source_tier_v7_expands_origin_checks_with_verified_backup( with sqlite3.connect(db_path) as conn: result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) assert result.from_version == 7 - assert result.to_version == SOURCE_SCHEMA_VERSION == 12 - assert result.applied_versions == (8, 9, 10, 11, 12) + assert result.to_version == SOURCE_SCHEMA_VERSION == 13 + assert result.applied_versions == (8, 9, 10, 11, 12, 13) assert conn.execute( """ SELECT predecessor_source_revision, predecessor_raw_id, baseline_raw_id, @@ -877,7 +892,7 @@ def test_source_tier_v2_migrates_to_v3_dropping_pending_blob_refs( assert result.from_version == 2 assert result.to_version == SOURCE_SCHEMA_VERSION - assert result.applied_versions == (3, 4, 5, 6, 7, 8, 9, 10, 11, 12) + assert result.applied_versions == (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13) assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == SOURCE_SCHEMA_VERSION assert not conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name='pending_blob_refs'" @@ -935,7 +950,7 @@ def test_source_tier_v3_adds_publication_reservations_with_verified_backup_recei conn = sqlite3.connect(db_path) try: result = migrate_archive_tier(conn, ArchiveTier.SOURCE, backup_manifest=manifest) - assert result.applied_versions == (4, 5, 6, 7, 8, 9, 10, 11, 12) + assert result.applied_versions == (4, 5, 6, 7, 8, 9, 10, 11, 12, 13) assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == SOURCE_SCHEMA_VERSION conn.execute( """ diff --git a/tests/unit/storage/test_raw_authority_ledger.py b/tests/unit/storage/test_raw_authority_ledger.py new file mode 100644 index 0000000000..5ceb3aaa75 --- /dev/null +++ b/tests/unit/storage/test_raw_authority_ledger.py @@ -0,0 +1,785 @@ +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import cast +from unittest.mock import patch + +import pytest + +from polylogue.config import Config +from polylogue.core.enums import Provider +from polylogue.core.json import JSONDocument, json_document +from polylogue.maintenance.models import MaintenanceCategory +from polylogue.sources.revision_backfill import census_historical_revision_evidence +from polylogue.storage import raw_authority as raw_authority_mod +from polylogue.storage import repair as repair_mod +from polylogue.storage.archive_readiness import raw_materialization_readiness_snapshot +from polylogue.storage.raw_authority import ( + RawReplayPlan, + RawReplayPlanOutcome, + RawReplayPlanStatus, + build_raw_replay_plans, + finalize_raw_authority_census, + read_raw_authority_census, + read_raw_authority_detail, + record_raw_authority_census, + record_raw_replay_outcome, + reject_stale_raw_replay_plan, + resolve_raw_authority_blocker, + validate_raw_replay_plan, +) +from polylogue.storage.repair import RepairResult, repair_raw_materialization +from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + +def _config(root: Path) -> Config: + return Config(archive_root=root, render_root=root / "render", sources=[], db_path=root / "archive.db") + + +def _read_detail_document(root: Path, query_handle: str, *, chunk_chars: int = 256) -> dict[str, object]: + chunks: list[str] = [] + handle: str | None = query_handle + digest: str | None = None + for _page in range(10_000): + assert handle is not None + page = read_raw_authority_detail(root, handle, chunk_chars=chunk_chars) + chunk = cast(str, page["chunk"]) + assert len(chunk) <= chunk_chars + chunks.append(chunk) + page_digest = cast(str, page["document_sha256"]) + digest = digest or page_digest + assert page_digest == digest + handle = cast(str | None, page["next_query_handle"]) + if handle is None: + break + else: + raise AssertionError("raw authority detail pagination did not terminate") + document = json.loads("".join(chunks)) + assert isinstance(document, dict) + return cast(dict[str, object], document) + + +def _write_codex_raw( + root: Path, + *, + native_id: str, + source_path: str, + acquired_at_ms: int, + text: str = "", +) -> str: + payload = ( + f'{{"type":"session_meta","payload":{{"id":"{native_id}"}}}}\n' + f'{{"type":"response_item","payload":{{"type":"message","id":"m-{acquired_at_ms}",' + f'"role":"user","content":[{{"type":"input_text","text":"{text}"}}]}}}}\n' + ).encode() + with ArchiveStore.open_existing(root, read_only=False) as archive: + return archive.write_raw_payload( + provider=Provider.CODEX, + payload=payload, + source_path=source_path, + acquired_at_ms=acquired_at_ms, + ) + + +def test_moved_path_census_stabilizes_preview_and_apply_plan_identity(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + old_raw = _write_codex_raw( + tmp_path, + native_id="moved-session", + source_path="old/location.jsonl", + acquired_at_ms=2, + text="old", + ) + new_raw = _write_codex_raw( + tmp_path, + native_id="moved-session", + source_path="new/location.jsonl", + acquired_at_ms=1, + text="new", + ) + # Prior history already knows the logical key at another path. The new + # raw begins as an uncensused singleton and must discover that history + # before an immutable plan is assigned. + census_historical_revision_evidence(tmp_path, selected_raw_ids=[old_raw]) + + preview = repair_raw_materialization(_config(tmp_path), dry_run=True, raw_artifact_limit=1) + applied = repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) + + assert len(preview.plan_outcomes) == len(applied.plan_outcomes) == 1 + assert preview.plan_outcomes[0].plan_id == applied.plan_outcomes[0].plan_id + assert set(preview.plan_outcomes[0].input_raw_ids) == {old_raw, new_raw} + assert set(applied.plan_outcomes[0].input_raw_ids) == {old_raw, new_raw} + + +def test_census_ledger_conserves_unselected_plan_and_application_receipt(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="first", source_path="first.jsonl", acquired_at_ms=1) + _write_codex_raw(tmp_path, native_id="second", source_path="second.jsonl", acquired_at_ms=2) + + incomplete = repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) + assert incomplete.census_receipt is not None + assert incomplete.census_receipt.quiescent is False + assert incomplete.census_receipt.plan_count == 0 + + result = repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) + + assert result.census_receipt is not None + assert result.census_receipt.plan_count == 2 + assert result.census_receipt.executable_plan_count == 2 + assert result.census_receipt.residual_plan_count == 0 + assert result.census_receipt.post_plan_count == 1 + assert result.census_receipt.post_inventory_digest is not None + assert result.census_receipt.post_inventory_digest != result.census_receipt.inventory_digest + assert result.census_receipt.lifecycle_status == "completed" + assert result.metrics["raw_materialization_plan_outcome_count"] == 2.0 + assert result.metrics["raw_materialization_plan_carried_forward_count"] == 1.0 + with sqlite3.connect(tmp_path / "source.db") as conn: + rows = conn.execute( + """ + SELECT selected, outcome_status, application_receipt_json + FROM raw_authority_census_plans + WHERE census_id = ? ORDER BY ordinal + """, + (result.census_receipt.census_id,), + ).fetchall() + plan_row = conn.execute( + """ + SELECT input_raw_ids_json, logical_keys_json, authority_witness_json, + source_preconditions_json, index_preconditions_json + FROM raw_authority_plans + WHERE plan_id = ( + SELECT plan_id FROM raw_authority_census_plans + WHERE census_id = ? AND selected = 1 + ) + """, + (result.census_receipt.census_id,), + ).fetchone() + assert {row[1] for row in rows} == {"executed", "carried_forward"} + executed = next(row for row in rows if row[1] == "executed") + assert executed[0] == 1 + assert '"application_rows"' in executed[2] + assert '"membership_rows"' in executed[2] + application_receipt = json.loads(executed[2]) + assert application_receipt["application_rows"] + assert application_receipt["head_rows"] + assert application_receipt["session_rows"] + assert plan_row is not None + assert all(value not in (None, "", "[]", "{}") for value in plan_row) + readiness = raw_materialization_readiness_snapshot(tmp_path) + census_status = cast(dict[str, object], readiness["raw_authority_census"]) + assert census_status["census_id"] == result.census_receipt.census_id + assert census_status["inventory_digest"] == result.census_receipt.inventory_digest + assert census_status["residual_digest"] == result.census_receipt.residual_digest + assert census_status["plan_count"] == 2 + assert census_status["executable_plan_count"] == 2 + assert census_status["residual_plan_count"] == 0 + assert census_status["lifecycle_status"] == "completed" + assert census_status["query_handle"] == result.census_receipt.query_handle + first_page = read_raw_authority_census(tmp_path, result.census_receipt.query_handle, limit=1) + assert first_page["returned_count"] == 1 + assert first_page["next_query_handle"] is not None + second_page = read_raw_authority_census(tmp_path, cast(str, first_page["next_query_handle"]), limit=1) + assert second_page["returned_count"] == 1 + assert second_page["next_query_handle"] is None + assert { + cast(dict[str, object], item)["outcome_status"] + for item in (*cast(list[object], first_page["plans"]), *cast(list[object], second_page["plans"])) + } == {"executed", "carried_forward"} + first_item = cast(dict[str, object], cast(list[object], first_page["plans"])[0]) + assert "application_receipt" not in first_item + assert "input_raw_ids" not in cast(dict[str, object], first_item["plan"]) + detail = _read_detail_document(tmp_path, cast(str, first_item["detail_query_handle"])) + assert cast(dict[str, object], detail["plan"])["input_raw_ids"] + assert cast(dict[str, object], first_page["census"])["post_plan_count"] == 1 + assert len(cast(list[object], first_page["post_plans"])) == 1 + + +def test_two_successive_quiescent_censuses_are_required_for_fixed_point(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="fixed", source_path="fixed.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).repaired_count == 1 + + first_empty = repair_raw_materialization(_config(tmp_path), dry_run=True) + second_empty = repair_raw_materialization(_config(tmp_path), dry_run=True) + + assert first_empty.census_receipt is not None + assert second_empty.census_receipt is not None + assert first_empty.census_receipt.fixed_point is False + assert second_empty.census_receipt.fixed_point is True + assert first_empty.census_receipt.inventory_digest == second_empty.census_receipt.inventory_digest + assert first_empty.census_receipt.residual_digest == second_empty.census_receipt.residual_digest + + +def test_stale_plan_persists_blocker_before_automatic_replay_refuses_work(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="stale", source_path="stale.jsonl", acquired_at_ms=1) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={"test": "stale"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET source_path = 'moved-after-plan.jsonl' WHERE raw_id = ?", (raw_id,)) + conn.commit() + + valid, observed = validate_raw_replay_plan(tmp_path, plan) + assert valid is False + outcome = reject_stale_raw_replay_plan(tmp_path, census.census_id, plan, observed) + + assert outcome.status.value == "rejected_stale" + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + assert ( + conn.execute( + "SELECT outcome_status FROM raw_authority_census_plans WHERE census_id = ? AND plan_id = ?", + (census.census_id, plan.plan_id), + ).fetchone()[0] + == "rejected_stale" + ) + refused = repair_raw_materialization(_config(tmp_path)) + assert refused.success is False + assert refused.metrics["raw_materialization_unresolved_blocker_count"] == 1.0 + + +def test_interrupted_census_has_no_partial_plan_visibility_and_retries_once(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + first = _write_codex_raw(tmp_path, native_id="atomic-first", source_path="atomic-first.jsonl", acquired_at_ms=1) + second = _write_codex_raw( + tmp_path, + native_id="atomic-second", + source_path="atomic-second.jsonl", + acquired_at_ms=2, + ) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[first, second]) + plans = build_raw_replay_plans(tmp_path, ((first,), (second,))) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + """ + CREATE TRIGGER abort_second_census_plan + BEFORE INSERT ON raw_authority_census_plans + WHEN NEW.ordinal = 1 + BEGIN + SELECT RAISE(ABORT, 'synthetic census interruption'); + END + """ + ) + conn.commit() + + with pytest.raises(sqlite3.IntegrityError, match="synthetic census interruption"): + record_raw_authority_census( + tmp_path, + plans, + selected_plan_ids={plan.plan_id for plan in plans}, + mode="apply", + quiescent=True, + scope={"test": "interruption"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM raw_authority_plans").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM raw_authority_census_plans").fetchone()[0] == 0 + conn.execute("DROP TRIGGER abort_second_census_plan") + conn.commit() + + receipt = record_raw_authority_census( + tmp_path, + plans, + selected_plan_ids={plan.plan_id for plan in plans}, + mode="apply", + quiescent=True, + scope={"test": "interruption"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0] == 1 + assert ( + conn.execute( + "SELECT COUNT(*) FROM raw_authority_census_plans WHERE census_id = ?", + (receipt.census_id,), + ).fetchone()[0] + == 2 + ) + for plan in plans: + record_raw_replay_outcome( + tmp_path, + receipt.census_id, + RawReplayPlanOutcome( + plan.plan_id, + plan.input_raw_ids, + RawReplayPlanStatus.RETRYABLE, + "test interruption recovered", + "retry", + ), + ) + finalized = finalize_raw_authority_census( + tmp_path, + receipt.census_id, + post_plans=plans, + post_residual={}, + interrupted=True, + ) + assert finalized.lifecycle_status == "interrupted" + + +def test_global_census_quiesces_moved_component_before_any_plan_is_published(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + first = _write_codex_raw( + tmp_path, + native_id="merged", + source_path="merged-old.jsonl", + acquired_at_ms=1, + text="old", + ) + second = _write_codex_raw( + tmp_path, + native_id="merged", + source_path="merged-new.jsonl", + acquired_at_ms=2, + text="new", + ) + third = _write_codex_raw( + tmp_path, + native_id="independent", + source_path="independent.jsonl", + acquired_at_ms=3, + ) + + incomplete_receipts = [] + for _expected_pass in range(2): + incomplete = repair_raw_materialization(_config(tmp_path), dry_run=True, raw_artifact_limit=1) + assert incomplete.census_receipt is not None + assert incomplete.census_receipt.quiescent is False + assert incomplete.census_receipt.plan_count == 0 + assert incomplete.metrics["raw_materialization_census_component_limit"] == 1.0 + assert incomplete.metrics["raw_materialization_census_components_attempted"] == 1.0 + incomplete_ledger = read_raw_authority_census(tmp_path, incomplete.census_receipt.query_handle) + assert incomplete_ledger["plans"] == [] + census_detail = _read_detail_document( + tmp_path, + cast(str, cast(dict[str, object], incomplete_ledger["census"])["detail_query_handle"]), + ) + pending_residual = cast(dict[str, object], census_detail["residual"]) + assert cast(int, pending_residual["census_pending_raw_count"]) >= 1 + assert len(cast(str, pending_residual["census_pending_raw_digest"])) == 64 + assert "census_pending_raw_ids" not in pending_residual + incomplete_receipts.append(incomplete.census_receipt.census_id) + assert len(set(incomplete_receipts)) == 2 + + preview = repair_raw_materialization(_config(tmp_path), dry_run=True, raw_artifact_limit=1) + + assert preview.census_receipt is not None + assert preview.census_receipt.quiescent is True + ledger = read_raw_authority_census(tmp_path, preview.census_receipt.query_handle) + raw_sets = { + frozenset( + cast( + list[str], + cast( + dict[str, object], + _read_detail_document( + tmp_path, + cast(str, cast(dict[str, object], item)["detail_query_handle"]), + )["plan"], + )["input_raw_ids"], + ) + ) + for item in cast(list[object], ledger["plans"]) + } + assert raw_sets == {frozenset((first, second)), frozenset((third,))} + + +def test_census_page_bounds_one_oversized_plan_and_detail_chunks_reconstruct_it(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_ids = tuple(f"raw-{index:05d}" for index in range(2_000)) + plan = RawReplayPlan( + plan_id="raw-replay:oversized", + input_digest="a" * 64, + input_raw_ids=raw_ids, + logical_keys=tuple(f"codex:key-{index:05d}" for index in range(2_000)), + authority_witness=json_document({"rows": [{"raw_id": raw_id} for raw_id in raw_ids]}), + source_preconditions=json_document({"rows": [{"raw_id": raw_id} for raw_id in raw_ids]}), + index_preconditions=json_document({"rows": [{"raw_id": raw_id} for raw_id in raw_ids]}), + ) + receipt = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids=set(), + executable_plan_ids={plan.plan_id}, + mode="dry_run", + quiescent=True, + scope={"test": "oversized"}, + residual={}, + ) + + page = read_raw_authority_census(tmp_path, receipt.query_handle, limit=1) + + assert len(json.dumps(page)) < 8_000 + item = cast(dict[str, object], cast(list[object], page["plans"])[0]) + summary = cast(dict[str, object], item["plan"]) + assert summary["input_raw_count"] == 2_000 + assert "input_raw_ids" not in summary + detail = _read_detail_document(tmp_path, cast(str, item["detail_query_handle"])) + detail_plan = cast(dict[str, object], detail["plan"]) + assert detail_plan["input_raw_ids"] == list(raw_ids) + assert len(cast(list[object], cast(dict[str, object], detail_plan["authority_witness"])["rows"])) == 2_000 + + +def test_interrupted_apply_recovers_exact_durable_postconditions(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="crash", source_path="crash.jsonl", acquired_at_ms=1) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=RuntimeError("synthetic crash")): + with pytest.raises(RuntimeError, match="synthetic crash"): + repair_raw_materialization(_config(tmp_path)) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_censuses WHERE lifecycle_status = 'planned'").fetchone()[0] + == 1 + ) + recovered = repair_raw_materialization(_config(tmp_path)) + assert recovered.metrics["raw_materialization_recovered_census_count"] == 1.0 + with sqlite3.connect(tmp_path / "source.db") as conn: + row = conn.execute( + """ + SELECT c.lifecycle_status, cp.outcome_status, cp.application_receipt_json + FROM raw_authority_censuses AS c + JOIN raw_authority_census_plans AS cp ON cp.census_id = c.census_id + WHERE c.lifecycle_status = 'interrupted' + """ + ).fetchone() + assert row is not None + assert row[:2] == ("interrupted", "executed") + recovered_receipt = json.loads(row[2]) + assert isinstance(recovered_receipt["application_rows"], list) + assert isinstance(recovered_receipt["membership_rows"], list) + assert recovered_receipt["application_rows"] or recovered_receipt["membership_rows"] + assert recovered_receipt["head_rows"] + assert recovered_receipt["session_rows"] + + +def test_parsed_timestamp_without_exact_application_receipt_fails_closed(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw(tmp_path, native_id="receipt", source_path="receipt.jsonl", acquired_at_ms=1) + real_receipt = raw_authority_mod.raw_replay_application_receipt + + def incomplete_receipt(root: Path, plan: RawReplayPlan) -> JSONDocument: + payload = dict(real_receipt(root, plan)) + payload["head_rows"] = [] + return json_document(payload) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=incomplete_receipt): + result = repair_raw_materialization(_config(tmp_path)) + + assert result.plan_outcomes[0].status is RawReplayPlanStatus.REJECTED_STALE + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + + +@pytest.mark.parametrize("field", ["session_id", "accepted_raw_id", "accepted_content_hash"]) +def test_application_receipt_requires_exact_application_authority(tmp_path: Path, field: str) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id=f"exact-{field}", source_path=f"{field}.jsonl", acquired_at_ms=1) + assert repair_raw_materialization(_config(tmp_path)).success is True + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + receipt = dict(raw_authority_mod.raw_replay_application_receipt(tmp_path, plan)) + application_rows = cast(list[dict[str, object]], receipt["application_rows"]) + assert application_rows + application_rows[0][field] = f"wrong-{field}" + + valid, problems = raw_authority_mod.validate_raw_replay_application_receipt(plan, receipt) + + assert valid is False + assert any("no application accepted authority matches" in problem for problem in problems) + + +def test_recovery_rejects_partial_expanded_membership_postconditions(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _write_codex_raw( + tmp_path, + native_id="partial-component", + source_path="partial-old.jsonl", + acquired_at_ms=1, + text="old", + ) + second = _write_codex_raw( + tmp_path, + native_id="partial-component", + source_path="partial-new.jsonl", + acquired_at_ms=2, + text="new", + ) + + with patch.object(repair_mod, "raw_replay_application_receipt", side_effect=RuntimeError("synthetic crash")): + with pytest.raises(RuntimeError, match="synthetic crash"): + repair_raw_materialization(_config(tmp_path)) + + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("DELETE FROM raw_session_memberships WHERE raw_id = ?", (second,)) + conn.commit() + + recovered = repair_raw_materialization(_config(tmp_path)) + + assert recovered.success is False + assert recovered.metrics["raw_materialization_unresolved_blocker_count"] == 1.0 + with sqlite3.connect(tmp_path / "source.db") as conn: + row = conn.execute( + """ + SELECT cp.outcome_status + FROM raw_authority_census_plans AS cp + JOIN raw_authority_censuses AS c ON c.census_id = cp.census_id + WHERE c.lifecycle_status = 'interrupted' AND cp.selected = 1 + """ + ).fetchone() + assert row == ("rejected_stale",) + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + + +def test_stale_blocker_resolution_replans_current_evidence_and_resumes(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="resume", source_path="resume.jsonl", acquired_at_ms=1) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={"test": "resolve"}, + residual={}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET source_path = 'resume-moved.jsonl' WHERE raw_id = ?", (raw_id,)) + conn.commit() + valid, observed = validate_raw_replay_plan(tmp_path, plan) + assert valid is False + rejected = reject_stale_raw_replay_plan(tmp_path, census.census_id, plan, observed) + blocker_id = cast(str, cast(dict[str, object], rejected.application_receipt)["blocker_id"]) + + census_page = read_raw_authority_census(tmp_path, census.query_handle) + plan_summary = cast(dict[str, object], cast(list[object], census_page["plans"])[0]) + first_detail_page = read_raw_authority_detail( + tmp_path, + cast(str, plan_summary["detail_query_handle"]), + chunk_chars=256, + ) + stale_continuation = cast(str, first_detail_page["next_query_handle"]) + + resolution = resolve_raw_authority_blocker(tmp_path, blocker_id, resolution="current path is authoritative") + with pytest.raises(RuntimeError, match="raw authority detail changed"): + read_raw_authority_detail(tmp_path, stale_continuation, chunk_chars=256) + current_detail = _read_detail_document(tmp_path, cast(str, resolution["detail_query_handle"])) + resumed = repair_raw_materialization(_config(tmp_path)) + + assert resolution["blocker_id"] == blocker_id + resolution_plan = cast(dict[str, object], resolution["current_plan"]) + assert resolution_plan["input_raw_count"] == 1 + assert "input_raw_ids" not in resolution_plan + stored_resolution = cast( + dict[str, object], + cast(dict[str, object], cast(list[object], current_detail["blockers"])[0])["resolution"], + ) + assert cast(dict[str, object], stored_resolution["current_plan"])["input_raw_ids"] == [raw_id] + assert resumed.success is True + assert resumed.repaired_count == 1 + assert "raw_materialization_unresolved_blocker_count" not in resumed.metrics + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 0 + ) + + +def test_identical_stale_rejection_after_resolution_creates_new_open_blocker(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="repeat", source_path="repeat.jsonl", acquired_at_ms=1) + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) + plan = build_raw_replay_plans(tmp_path, ((raw_id,),))[0] + + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET source_path = 'repeat-moved.jsonl' WHERE raw_id = ?", (raw_id,)) + conn.commit() + valid, observed = validate_raw_replay_plan(tmp_path, plan) + assert valid is False + + first_census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={"test": "repeat-1"}, + residual={}, + ) + first = reject_stale_raw_replay_plan(tmp_path, first_census.census_id, plan, observed) + first_blocker = cast(str, cast(dict[str, object], first.application_receipt)["blocker_id"]) + resolve_raw_authority_blocker(tmp_path, first_blocker, resolution="acknowledge first occurrence") + + second_census = record_raw_authority_census( + tmp_path, + (plan,), + selected_plan_ids={plan.plan_id}, + mode="apply", + quiescent=True, + scope={"test": "repeat-2"}, + residual={}, + ) + second = reject_stale_raw_replay_plan(tmp_path, second_census.census_id, plan, observed) + second_blocker = cast(str, cast(dict[str, object], second.application_receipt)["blocker_id"]) + + assert second_blocker != first_blocker + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM raw_authority_blockers WHERE resolved_at_ms IS NULL").fetchone()[0] == 1 + ) + page = read_raw_authority_census(tmp_path, second_census.query_handle) + assert page["blocker_count"] == 1 + item = cast(dict[str, object], cast(list[object], page["plans"])[0]) + detail = _read_detail_document(tmp_path, cast(str, item["detail_query_handle"])) + blockers = cast(list[object], detail["blockers"]) + assert len(blockers) == 1 + assert cast(dict[str, object], blockers[0])["blocker_id"] == second_blocker + + +def test_fixed_point_compares_residual_identity_and_parser_fingerprint(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + first = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["a"]}, + ) + second = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["b"]}, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + "UPDATE raw_authority_censuses SET parser_fingerprint = 'stale-parser' WHERE census_id = ?", + (second.census_id,), + ) + conn.commit() + third = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["b"]}, + ) + fourth = record_raw_authority_census( + tmp_path, + (), + selected_plan_ids=set(), + executable_plan_ids=set(), + mode="dry_run", + quiescent=True, + scope={"test": "fixed-point"}, + residual={"missing_blob_raw_ids": ["b"]}, + ) + assert first.fixed_point is False + assert second.fixed_point is False + assert third.fixed_point is False + assert fourth.fixed_point is True + + +def test_stale_per_raw_parser_fingerprint_is_recensused_before_planning(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + raw_id = _write_codex_raw(tmp_path, native_id="parser-drift", source_path="parser-drift.jsonl", acquired_at_ms=1) + first = repair_raw_materialization(_config(tmp_path), dry_run=True) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute( + "UPDATE raw_authority_parser_census SET parser_fingerprint = 'old-parser' WHERE raw_id = ?", + (raw_id,), + ) + conn.commit() + + second = repair_raw_materialization(_config(tmp_path), dry_run=True) + + assert first.plan_outcomes[0].plan_id == second.plan_outcomes[0].plan_id + with sqlite3.connect(tmp_path / "source.db") as conn: + assert ( + conn.execute( + "SELECT parser_fingerprint FROM raw_authority_parser_census WHERE raw_id = ?", + (raw_id,), + ).fetchone()[0] + == "revision-membership-v1" + ) + + +def test_repair_result_bounds_public_plan_outcomes() -> None: + outcomes = tuple( + RawReplayPlanOutcome( + f"plan-{index}", + (f"raw-{index}",), + RawReplayPlanStatus.RETRYABLE, + "test", + "retry", + ) + for index in range(10) + ) + result = RepairResult( + "raw_materialization", + MaintenanceCategory.DERIVED_REPAIR, + False, + 0, + False, + plan_outcomes=outcomes, + ).to_dict() + assert result["plan_outcome_count"] == 10 + assert len(cast(list[object], result["plan_outcomes"])) == 8 + assert result["plan_outcomes_truncated"] is True + + +def test_repair_result_omits_unbounded_receipt_rows_from_outcome_sample() -> None: + outcome = RawReplayPlanOutcome( + "plan-with-receipt", + tuple(f"raw-{index}" for index in range(100)), + RawReplayPlanStatus.EXECUTED, + "done", + "none", + json_document({"application_rows": [{"row": index} for index in range(1000)]}), + ) + result = RepairResult( + "raw_materialization", + MaintenanceCategory.DERIVED_REPAIR, + False, + 1, + True, + plan_outcomes=(outcome,), + ).to_dict() + sample = cast(list[dict[str, object]], result["plan_outcomes"])[0] + assert sample["has_application_receipt"] is True + assert "application_receipt" not in sample + assert sample["input_raw_count"] == 100 + assert len(cast(list[object], sample["input_raw_id_sample"])) == 8 + assert sample["input_raw_id_sample_truncated"] is True + assert "input_raw_ids" not in sample diff --git a/tests/unit/storage/test_repair.py b/tests/unit/storage/test_repair.py index 4ea5adbd21..cdb07582b0 100644 --- a/tests/unit/storage/test_repair.py +++ b/tests/unit/storage/test_repair.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import sqlite3 from collections.abc import Iterator from contextlib import contextmanager @@ -12,6 +11,7 @@ from polylogue.config import Config from polylogue.maintenance.models import DerivedModelStatus +from polylogue.sources.revision_backfill import census_historical_revision_evidence from polylogue.storage import repair as repair_mod from polylogue.storage.blob_store import BlobStore from polylogue.storage.insights.session.repair_assessment import assess_session_insight_repairs @@ -24,6 +24,21 @@ def _config(tmp_path: Path) -> Config: return Config(archive_root=tmp_path, render_root=tmp_path, sources=[], db_path=tmp_path / "archive.db") +def _complete_bounded_raw_census(config: Config, *, limit: int) -> tuple[repair_mod.RepairResult, list[str]]: + """Advance census-only passes until a quiescent preview can publish plans.""" + incomplete_census_ids: list[str] = [] + for _pass in range(1_000): + result = repair_mod.repair_raw_materialization(config, dry_run=True, raw_artifact_limit=limit) + assert result.census_receipt is not None + if result.census_receipt.quiescent: + return result, incomplete_census_ids + assert result.census_receipt.plan_count == 0 + attempted = result.metrics["raw_materialization_census_components_attempted"] + assert 1.0 <= attempted <= float(limit) + incomplete_census_ids.append(result.census_receipt.census_id) + raise AssertionError("bounded raw census did not quiesce") + + def _status( *, source_documents: int = 0, @@ -257,6 +272,8 @@ def test_raw_materialization_preview_counts_replayable_rows_without_erasing_miss "raw_materialization_before_component_count": 1.0, "raw_materialization_selected_executable_component_count": 1.0, "raw_materialization_selected_blocked_component_count": 0.0, + "raw_materialization_census_sequence": 1.0, + "raw_materialization_census_fixed_point": 0.0, } assert "per-session revision authority" in result.detail assert "selected raw payload bytes total=" in result.detail @@ -1153,12 +1170,6 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) - with sqlite3.connect(tmp_path / "source.db") as source_conn: - source_authority_before = source_conn.execute( - "SELECT revision_authority FROM raw_sessions WHERE raw_id = ?", - (raw_id,), - ).fetchone() - result = repair_mod.repair_raw_materialization(config, dry_run=False) assert result.success is False @@ -1188,7 +1199,10 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: "SELECT parsed_at_ms, parse_error, revision_authority FROM raw_sessions WHERE raw_id = ?", (raw_id,), ).fetchone() - assert raw_state == (None, None, source_authority_before[0]) + # The source-only census now completes before replay planning. It may + # establish byte-proven source authority, while the incomparable index + # state still remains untouched and receives a deferred application. + assert raw_state == (None, None, "byte_proven") with sqlite3.connect(tmp_path / "index.db") as index_conn: deferred = index_conn.execute( "SELECT decision, detail FROM raw_revision_applications WHERE raw_id = ?", @@ -1213,31 +1227,31 @@ def __init__(self, *_args: object, **_kwargs: object) -> None: def test_raw_materialization_dry_run_reports_limited_selection( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + sizes = [512, 1024, 2048, 4096] + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_ids = [ + archive.write_raw_payload( + provider=Provider.CODEX, + payload=f'{{"type":"session_meta","payload":{{"id":"dry-{index}"}}}}\n'.encode(), + source_path=f"dry-{index}.jsonl", + acquired_at_ms=index + 1, + ) + for index in range(4) + ] + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.executemany("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", zip(sizes, raw_ids, strict=True)) + conn.commit() config = _config(tmp_path) - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-slow", "raw-2", "raw-3", "raw-4"], - 0, - 4, - { - "raw-slow": 512, - "raw-2": 1024, - "raw-3": 2048, - "raw-4": 4096, - }, - ), - ) - result = repair_mod.repair_raw_materialization( - config, - dry_run=True, - raw_artifact_limit=2, - ) + result, incomplete_censuses = _complete_bounded_raw_census(config, limit=2) + assert len(incomplete_censuses) == 1 assert result.success is False assert result.repaired_count == 0 assert "Would: classify and replay" in result.detail @@ -1251,61 +1265,37 @@ def test_raw_materialization_dry_run_reports_limited_selection( def test_raw_materialization_execute_limits_authority_selection( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - config = _config(tmp_path) - calls: dict[str, object] = {} - - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - calls["db_path"] = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - calls["archive_root"] = archive_root - - async def close(self) -> None: - calls["closed"] = True - - class FakeParseResult: - processed_ids = {"session-2", "session-3"} - parse_failures = 0 - - class FakeParsingService: - def __init__(self, **_kwargs: object) -> None: - pass - - async def parse_from_raw(self, **kwargs: object) -> FakeParseResult: - calls["parse_kwargs"] = kwargs - return FakeParseResult() + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-slow", "raw-2", "raw-3", "raw-4"], - 0, - 4, - { - "raw-slow": 512, - "raw-2": 1024, - "raw-3": 2048, - "raw-4": 4096, - }, - ), - ) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_ids = [ + archive.write_raw_payload( + provider=Provider.CODEX, + payload=f'{{"type":"session_meta","payload":{{"id":"execute-{index}"}}}}\n'.encode(), + source_path=f"execute-{index}.jsonl", + acquired_at_ms=index + 1, + ) + for index in range(4) + ] + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.executemany( + "UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", + zip((512, 1024, 2048, 4096), raw_ids, strict=True), + ) + conn.commit() + config = _config(tmp_path) - result = repair_mod.repair_raw_materialization( - config, - raw_artifact_limit=2, - ) + preview, incomplete_censuses = _complete_bounded_raw_census(config, limit=2) + result = repair_mod.repair_raw_materialization(config, raw_artifact_limit=2) + assert len(incomplete_censuses) == 1 + assert len(preview.plan_outcomes) == 2 assert result.success is False - assert result.repaired_count == 0 - assert "parse_kwargs" not in calls + assert result.repaired_count == 2 assert result.metrics["raw_materialization_candidate_count"] == 4.0 assert result.metrics["raw_materialization_selected_count"] == 2.0 assert result.metrics["raw_materialization_executed_count"] == 2.0 @@ -1564,116 +1554,66 @@ def test_raw_materialization_uses_authority_substrate_not_legacy_ingest_stage( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - config = _config(tmp_path) - calls: dict[str, object] = {} - - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - calls["db_path"] = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - calls["archive_root"] = archive_root - - async def close(self) -> None: - calls["closed"] = True + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root - class FakeParseResult: - processed_ids = {"session-1", "session-2"} - parse_failures = 0 + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"authority-substrate"}}\n', + source_path="authority-substrate.jsonl", + acquired_at_ms=1, + ) + config = _config(tmp_path) - class FakeParsingService: + class UnexpectedParsingService: def __init__(self, **_kwargs: object) -> None: - pass + pytest.fail("raw authority repair must not construct the legacy ParsingService") - async def parse_from_raw(self, **kwargs: object) -> FakeParseResult: - calls["parse_kwargs"] = kwargs - return FakeParseResult() - - def fake_candidate_ids( - _config: Config, - *, - raw_artifact_id: str | None = None, - provider: str | None = None, - source_family: str | None = None, - source_root: Path | None = None, - ) -> repair_mod.RawMaterializationCandidates: - calls["raw_artifact_id"] = raw_artifact_id - calls["provider"] = provider - calls["source_family"] = source_family - calls["source_root"] = source_root - return repair_mod.RawMaterializationCandidates(["raw-1"], 0, 0) - - monkeypatch.setattr(repair_mod, "_raw_materialization_candidate_ids", fake_candidate_ids) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) + monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) result = repair_mod.repair_raw_materialization(config, dry_run=False) - assert result.success is False - assert result.repaired_count == 0 + assert result.success is True + assert result.repaired_count == 1 assert "typed revision authority" in result.detail - assert "parse_kwargs" not in calls - assert calls["raw_artifact_id"] is None - assert "closed" not in calls def test_raw_materialization_reports_authority_progress_and_payload_size( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"progress"}}\n', + source_path="progress.jsonl", + acquired_at_ms=1, + ) + declared_size = 256 * 1024 * 1024 + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", (declared_size, raw_id)) + conn.commit() config = _config(tmp_path) progress: list[str] = [] - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - self.db_path = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - self.backend = backend - self.archive_root = archive_root - - async def close(self) -> None: - pass - - class FakeParseResult: - processed_ids = {"session-1"} - parse_failures = 0 - - class FakeParsingService: - def __init__(self, **_kwargs: object) -> None: - pass - - async def parse_from_raw(self, **_kwargs: object) -> FakeParseResult: - return FakeParseResult() - - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-1"], - 0, - 0, - {"raw-1": 256 * 1024 * 1024}, - ), - ) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) - result = repair_mod.repair_raw_materialization( config, dry_run=False, progress_callback=lambda _amount, desc=None: progress.append(desc or ""), ) - assert result.success is False + assert result.success is True assert len(progress) == 1 assert "typed revision authority" in progress[0] - assert result.metrics["raw_materialization_total_blob_bytes"] == float(256 * 1024 * 1024) - assert result.metrics["raw_materialization_max_blob_bytes"] == float(256 * 1024 * 1024) + assert result.metrics["raw_materialization_total_blob_bytes"] == float(declared_size) + assert result.metrics["raw_materialization_max_blob_bytes"] == float(declared_size) assert result.metrics["raw_materialization_selected_count"] == 1.0 @@ -1681,32 +1621,35 @@ def test_raw_materialization_blocks_oversized_actual_replay( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CHATGPT, + payload=b"{}", + source_path="oversized.json", + acquired_at_ms=1, + ) + oversized = 2 * 1024 * 1024 * 1024 + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", (oversized, raw_id)) + conn.commit() config = _config(tmp_path) class UnexpectedParsingService: def __init__(self, **_kwargs: object) -> None: raise AssertionError("oversized raw rows should be blocked before parsing") - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-1"], - 0, - 0, - {"raw-1": 2 * 1024 * 1024 * 1024}, - expanded_raw_ids=("raw-1",), - expanded_blob_bytes={"raw-1": 2 * 1024 * 1024 * 1024}, - authority_components=(("raw-1",),), - ), - ) monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) result = repair_mod.repair_raw_materialization(config, dry_run=False) assert result.success is False assert result.repaired_count == 0 - assert "1 replay candidate(s) remain" in result.detail + assert "planning paused" in result.detail assert result.metrics["raw_materialization_oversized_count"] == 1.0 assert result.metrics["raw_materialization_resource_blocked_count"] == 1.0 assert result.metrics["raw_materialization_executed_count"] == 0.0 @@ -1717,6 +1660,23 @@ def test_raw_materialization_classifies_oversized_stream_record_replay( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b'{"type":"session_meta","payload":{"id":"oversized-stream"}}\n', + source_path="/captures/codex/session.jsonl", + acquired_at_ms=1, + ) + oversized = 2 * 1024 * 1024 * 1024 + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET blob_size = ? WHERE raw_id = ?", (oversized, raw_id)) + conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=[raw_id]) config = _config(tmp_path) calls: dict[str, object] = {} @@ -1743,21 +1703,6 @@ async def parse_from_raw(self, **kwargs: object) -> FakeParseResult: calls["parse_kwargs"] = kwargs return FakeParseResult() - monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates( - ["raw-1"], - 0, - 0, - {"raw-1": 2 * 1024 * 1024 * 1024}, - {"raw-1": "claude-code-session"}, - {"raw-1": "/captures/claude/session.jsonl"}, - expanded_raw_ids=("raw-1",), - expanded_blob_bytes={"raw-1": 2 * 1024 * 1024 * 1024}, - authority_components=(("raw-1",),), - ), - ) monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) @@ -1807,6 +1752,7 @@ def test_raw_materialization_blocks_oversized_expanded_cohort_before_blob_open( (repair_mod.RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + 1, oversized_raw), ) source_conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=[small_raw, oversized_raw]) monkeypatch.setattr( "polylogue.sources.revision_backfill._parse_retained_raw", @@ -1854,6 +1800,7 @@ def test_raw_materialization_backlog_expands_to_oversized_materialized_sibling( ("large-done", "codex-session", large_raw, "done", bytes(32)), ) index_conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=[small_raw, large_raw]) backlog = repair_mod.raw_materialization_replay_backlog(_config(tmp_path)) assert backlog["candidate_count"] == 1 @@ -1896,6 +1843,7 @@ def test_raw_materialization_blocks_aggregate_sub_limit_cohort_before_blob_open( ((per_raw_size, raw_id) for raw_id in raw_ids), ) source_conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=raw_ids) backlog = repair_mod.raw_materialization_replay_backlog(_config(tmp_path)) assert backlog["oversized_count"] == 0 @@ -1951,6 +1899,9 @@ def test_raw_materialization_processes_independent_components_across_bounded_pas assert backlog["execution_blocked"] is False assert backlog["executable_authority_component_count"] == raw_count + preview, incomplete_censuses = _complete_bounded_raw_census(config, limit=5) + assert len(incomplete_censuses) == 4 + assert len(preview.plan_outcomes) == 5 repaired_per_pass: list[int] = [] for _pass in range(5): result = repair_mod.repair_raw_materialization(config, raw_artifact_limit=5) @@ -1959,8 +1910,8 @@ def test_raw_materialization_processes_independent_components_across_bounded_pas assert repair_mod.repair_raw_materialization(config, raw_artifact_limit=5).success is True -def test_raw_materialization_receipts_rotate_retryable_plan_behind_unattempted_work(tmp_path: Path) -> None: - """A retryable oldest component must not monopolize a bounded daemon slot.""" +def test_raw_materialization_durable_ledger_survives_ops_reset_for_fairness(tmp_path: Path) -> None: + """A retryable oldest component must not monopolize a slot after ops reset.""" from polylogue.core.enums import Provider from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -1985,15 +1936,11 @@ def test_raw_materialization_receipts_rotate_retryable_plan_behind_unattempted_w (repair_mod.RAW_MATERIALIZATION_EXECUTE_BLOB_LIMIT_BYTES + 1, raw_ids[0]), ) conn.commit() + census_historical_revision_evidence(tmp_path, selected_raw_ids=raw_ids) first = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) assert first.plan_outcomes[0].status.value == "retryable" - with sqlite3.connect(tmp_path / "ops.db") as conn: - conn.execute( - "INSERT INTO daemon_events(ts_ms, kind, payload_json) VALUES (1, 'raw_materialization_pass', ?)", - (json.dumps({"plan_outcomes": [outcome.to_dict() for outcome in first.plan_outcomes]}),), - ) - conn.commit() + (tmp_path / "ops.db").unlink() second = repair_mod.repair_raw_materialization(_config(tmp_path), raw_artifact_limit=1) assert second.repaired_count == 1 @@ -2095,13 +2042,18 @@ def test_raw_materialization_batch_limit_counts_authority_components(tmp_path: P assert before["candidate_count"] == 9 assert before["authority_component_count"] == 5 - preview = repair_mod.repair_raw_materialization(config, dry_run=True, raw_artifact_limit=3) + preview, incomplete_censuses = _complete_bounded_raw_census(config, limit=3) first = repair_mod.repair_raw_materialization(config, raw_artifact_limit=3) after = repair_mod.raw_materialization_replay_backlog(config) + # The first bounded attempt discovers the five-revision shared component + # transitively; the next pass handles the remaining independent components + # and publishes the complete plan inventory. + assert len(incomplete_censuses) == 1 assert first.repaired_count == 3, (first.detail, first.metrics, after) assert first.metrics["raw_materialization_selected_component_count"] == 3.0 - assert first.metrics["raw_materialization_plan_outcome_count"] == 3.0 + assert first.metrics["raw_materialization_plan_outcome_count"] == 5.0 + assert first.metrics["raw_materialization_plan_carried_forward_count"] == 2.0 assert first.metrics["raw_materialization_plan_executed_count"] == 3.0 assert {outcome.plan_id for outcome in first.plan_outcomes} == { outcome.plan_id for outcome in preview.plan_outcomes @@ -2114,45 +2066,38 @@ def test_raw_materialization_quarantines_parse_failures_without_legacy_parser( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - config = _config(tmp_path) - - class FakeBackend: - def __init__(self, *, db_path: Path) -> None: - self.db_path = db_path - - class FakeRepository: - def __init__(self, *, backend: FakeBackend, archive_root: Path) -> None: - self.backend = backend - self.archive_root = archive_root - - async def close(self) -> None: - pass + from polylogue.core.enums import Provider + from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root - class FakeParseResult: - processed_ids: set[str] = set() - parse_failures = 1 + initialize_active_archive_root(tmp_path) + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + raw_id = archive.write_raw_payload( + provider=Provider.CODEX, + payload=b"\xff\n", + source_path="broken.jsonl", + acquired_at_ms=1, + ) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_sessions SET parsed_at_ms = 1 WHERE raw_id = ?", (raw_id,)) + conn.commit() + config = _config(tmp_path) - class FakeParsingService: + class UnexpectedParsingService: def __init__(self, **_kwargs: object) -> None: - pass - - async def parse_from_raw(self, **_kwargs: object) -> FakeParseResult: - return FakeParseResult() + pytest.fail("parse failures must remain inside the authority census route") + monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", UnexpectedParsingService) monkeypatch.setattr( - repair_mod, - "_raw_materialization_candidate_ids", - lambda *_args, **_kwargs: repair_mod.RawMaterializationCandidates(["raw-1"], 0, 1), + "polylogue.sources.revision_backfill._parse_retained_raw", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("synthetic retained-byte decode failure")), ) - monkeypatch.setattr("polylogue.pipeline.services.parsing.ParsingService", FakeParsingService) - monkeypatch.setattr("polylogue.storage.repository.SessionRepository", FakeRepository) - monkeypatch.setattr("polylogue.storage.sqlite.async_sqlite.SQLiteBackend", FakeBackend) result = repair_mod.repair_raw_materialization(config, dry_run=False) assert result.success is False assert result.repaired_count == 0 - assert "typed revision authority" in result.detail + assert "parser census completes" in result.detail assert result.metrics["raw_materialization_already_parsed_count"] == 1.0