diff --git a/devtools/command_catalog.py b/devtools/command_catalog.py index 1841ad40cd..f52b08040f 100644 --- a/devtools/command_catalog.py +++ b/devtools/command_catalog.py @@ -292,6 +292,17 @@ def to_dict(self) -> dict[str, object]: use_when="Assert memory budgets around a concrete query or archive-facing command.", examples=("devtools bench memory --max-rss-mb 1536 -- polylogue --plain analyze",), ), + CommandSpec( + "bench query-envelope", + "benchmarking", + "Measure repeated incident-scale query RSS, PSS, swap, and temp envelopes.", + "devtools.query_execution_envelope", + json_flag=False, + use_when="Run the opt-in live archive proof for repeated aggregate query_units calls and emit a receipt.", + examples=( + "devtools bench query-envelope --archive-root /path/to/archive --receipt .cache/query-envelope.json", + ), + ), CommandSpec( "archive lineage-validation", "archive", diff --git a/devtools/query_execution_envelope.py b/devtools/query_execution_envelope.py new file mode 100644 index 0000000000..ceb88d8a23 --- /dev/null +++ b/devtools/query_execution_envelope.py @@ -0,0 +1,311 @@ +"""Measure repeated incident-scale query resource envelopes. + +The command is an opt-in lab check. It opens the supplied archive through the +public query route and never writes to the archive. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import threading +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from shutil import disk_usage +from typing import Any + +from polylogue import Polylogue +from polylogue.core.errors import SchemaVersionMismatchError + +DEFAULT_EXPRESSION = "actions where tool:shell | group by tool | count" +DEFAULT_ROUNDS = 20 +DEFAULT_WARMUP = 3 +DEFAULT_BASELINE = 5 +DEFAULT_TOLERANCE = 0.25 +DEFAULT_MAX_RSS_MB = 1536 +DEFAULT_MAX_PSS_MB = 1536 +DEFAULT_MAX_SWAP_GROWTH_MB = 64 +DEFAULT_MAX_TEMP_GROWTH_MB = 64 +MIB = 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class ResourceSample: + """One process and temporary-filesystem observation.""" + + rss_bytes: int + pss_bytes: int + swap_bytes: int + temp_delta_bytes: int + + +PROC_MEMORY_FIELDS = ("VmRSS", "Pss", "VmSwap") + + +class ResourceProbeUnavailableError(RuntimeError): + """procfs did not report a field the declared envelope is measured against.""" + + +def _parse_proc_memory(text: str) -> tuple[int, int, int]: + """Return RSS, PSS, and swap in bytes from concatenated procfs reports. + + A missing field is refused rather than defaulted to zero: a zero sample + satisfies every declared limit, so the envelope would pass without + measuring anything. + """ + values: dict[str, int] = {} + for line in text.splitlines(): + key, _, value = line.partition(":") + if key not in PROC_MEMORY_FIELDS: + continue + fields = value.split() + if not fields: + continue + values[key] = int(fields[0]) * 1024 + missing = [field for field in PROC_MEMORY_FIELDS if field not in values] + if missing: + raise ResourceProbeUnavailableError(f"procfs did not report {', '.join(missing)}") + return values["VmRSS"], values["Pss"], values["VmSwap"] + + +def _proc_memory() -> tuple[int, int, int]: + """Return current RSS, PSS, and swap from procfs for this process.""" + try: + status = Path("/proc/self/status").read_text(encoding="ascii") + smaps_rollup = Path("/proc/self/smaps_rollup").read_text(encoding="ascii") + except (OSError, ValueError) as exc: + raise ResourceProbeUnavailableError(f"procfs memory reports are unreadable: {exc}") from exc + return _parse_proc_memory(status + smaps_rollup) + + +def _temp_used_bytes(temp_root: Path) -> int: + """Return used bytes on the filesystem containing the temp root.""" + try: + usage = disk_usage(temp_root) + except OSError: + return 0 + return usage.total - usage.free + + +async def _query_once(archive: Polylogue, expression: str) -> dict[str, Any]: + envelope = await archive.query_units(expression, limit=100) + return envelope.model_dump(mode="json") + + +async def measure_query_envelope( + archive_root: Path, + *, + expression: str = DEFAULT_EXPRESSION, + rounds: int = DEFAULT_ROUNDS, + warmup: int = DEFAULT_WARMUP, + baseline_rounds: int = DEFAULT_BASELINE, + tolerance: float = DEFAULT_TOLERANCE, + sample_interval_s: float = 0.05, + max_rss_bytes: int = DEFAULT_MAX_RSS_MB * MIB, + max_pss_bytes: int = DEFAULT_MAX_PSS_MB * MIB, + max_swap_growth_bytes: int = DEFAULT_MAX_SWAP_GROWTH_MB * MIB, + max_temp_growth_bytes: int = DEFAULT_MAX_TEMP_GROWTH_MB * MIB, +) -> dict[str, Any]: + """Run repeated aggregate reads and return a resource receipt.""" + if rounds < 20: + raise ValueError("rounds must be at least 20") + if warmup < 0 or baseline_rounds < 1: + raise ValueError("warmup must be non-negative and baseline_rounds must be positive") + if tolerance < 0 or sample_interval_s < 0: + raise ValueError("tolerance and sample interval must be non-negative") + if min(max_rss_bytes, max_pss_bytes, max_swap_growth_bytes, max_temp_growth_bytes) < 0: + raise ValueError("resource envelope limits must be non-negative") + + archive_root = archive_root.resolve() + db_path = archive_root / "index.db" + if not db_path.is_file(): + raise FileNotFoundError(db_path) + + temp_root = Path(os.environ.get("TMPDIR", "/tmp")) + temp_before = _temp_used_bytes(temp_root) + initial_rss, initial_pss, initial_swap = _proc_memory() + initial = ResourceSample(initial_rss, initial_pss, initial_swap, 0) + peak = initial + stop = threading.Event() + + def observe() -> ResourceSample: + nonlocal peak + rss, pss, swap = _proc_memory() + candidate = ResourceSample(rss, pss, swap, max(0, _temp_used_bytes(temp_root) - temp_before)) + peak = ResourceSample( + max(peak.rss_bytes, candidate.rss_bytes), + max(peak.pss_bytes, candidate.pss_bytes), + max(peak.swap_bytes, candidate.swap_bytes), + max(peak.temp_delta_bytes, candidate.temp_delta_bytes), + ) + return candidate + + def sample() -> None: + # The measured loop observes every round on this thread too, so a + # persistent probe failure still reaches the caller. + while not stop.is_set(): + try: + observe() + except ResourceProbeUnavailableError: + return + time.sleep(sample_interval_s) + + sampler = threading.Thread(target=sample, name="query-envelope-sampler", daemon=True) + sampler.start() + started = time.perf_counter() + result_counts: list[int] = [] + samples: list[dict[str, Any]] = [] + try: + async with Polylogue(archive_root=archive_root, db_path=db_path) as archive: + for phase, count in (("warmup", warmup), ("baseline", baseline_rounds), ("measured", rounds)): + for round_number in range(count): + result_count = len((await _query_once(archive, expression)).get("items", [])) + result_counts.append(result_count) + sample_now = observe() + samples.append( + { + "phase": phase, + "round": round_number + 1, + "result_item_count": result_count, + **asdict(sample_now), + } + ) + quiescent = observe() + finally: + stop.set() + sampler.join(timeout=2) + + elapsed_ms = round((time.perf_counter() - started) * 1000, 3) + baseline = [ + ResourceSample(item["rss_bytes"], item["pss_bytes"], item["swap_bytes"], item["temp_delta_bytes"]) + for item in samples + if item["phase"] == "baseline" + ] + measured = [ + ResourceSample(item["rss_bytes"], item["pss_bytes"], item["swap_bytes"], item["temp_delta_bytes"]) + for item in samples + if item["phase"] == "measured" + ] + baseline_rss = max(sample.rss_bytes for sample in baseline) + baseline_pss = max(sample.pss_bytes for sample in baseline) + baseline_swap = max(sample.swap_bytes for sample in baseline) + baseline_temp = max(sample.temp_delta_bytes for sample in baseline) + final = measured[-3:] + return_checks = { + "rss": all(current.rss_bytes <= max(1, baseline_rss) * (1 + tolerance) for current in final), + "pss": all(current.pss_bytes <= max(1, baseline_pss) * (1 + tolerance) for current in final), + "swap": all(current.swap_bytes <= baseline_swap + max_swap_growth_bytes for current in final), + "temp": all(current.temp_delta_bytes <= baseline_temp + max_temp_growth_bytes for current in final), + } + absolute_checks = { + "rss": peak.rss_bytes <= max_rss_bytes, + "pss": peak.pss_bytes <= max_pss_bytes, + "swap": peak.swap_bytes <= initial_swap + max_swap_growth_bytes, + "temp": peak.temp_delta_bytes <= max_temp_growth_bytes, + } + returned = all(return_checks.values()) and all(absolute_checks.values()) + return { + "status": "succeeded" if returned else "failed", + "archive_root": str(archive_root), + "archive_generation": db_path.resolve().parent.name, + "archive_index_bytes": db_path.stat().st_size, + "expression": expression, + "rounds": rounds, + "warmup_rounds": warmup, + "baseline_rounds": baseline_rounds, + "tolerance": tolerance, + "declared_envelope": { + "max_rss_bytes": max_rss_bytes, + "max_pss_bytes": max_pss_bytes, + "max_swap_growth_bytes": max_swap_growth_bytes, + "max_temp_growth_bytes": max_temp_growth_bytes, + "return_tolerance": tolerance, + }, + "steady_state_baseline": { + "rss_bytes": baseline_rss, + "pss_bytes": baseline_pss, + "swap_bytes": baseline_swap, + "temp_delta_bytes": baseline_temp, + }, + "peak": asdict(peak), + "initial_sample": asdict(initial), + "quiescent_sample": asdict(quiescent), + "final_samples": [asdict(sample) for sample in final], + "samples": samples, + "return_checks": return_checks, + "absolute_checks": absolute_checks, + "result_item_counts": result_counts, + "elapsed_ms": elapsed_ms, + "returned_to_envelope": returned, + "regression_path": ( + "Rerun this command against the same promoted generation. A failed status identifies RSS/PSS, " + "swap, temp, or return-to-baseline drift; compare the named check and samples." + ), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archive-root", type=Path, required=True) + parser.add_argument("--rounds", type=int, default=DEFAULT_ROUNDS) + parser.add_argument("--warmup", type=int, default=DEFAULT_WARMUP) + parser.add_argument("--baseline-rounds", type=int, default=DEFAULT_BASELINE) + parser.add_argument("--tolerance", type=float, default=DEFAULT_TOLERANCE) + parser.add_argument("--sample-interval", type=float, default=0.05) + parser.add_argument("--max-rss-mb", type=int, default=DEFAULT_MAX_RSS_MB) + parser.add_argument("--max-pss-mb", type=int, default=DEFAULT_MAX_PSS_MB) + parser.add_argument("--max-swap-growth-mb", type=int, default=DEFAULT_MAX_SWAP_GROWTH_MB) + parser.add_argument("--max-temp-growth-mb", type=int, default=DEFAULT_MAX_TEMP_GROWTH_MB) + parser.add_argument("--expression", default=DEFAULT_EXPRESSION) + parser.add_argument("--receipt", type=Path) + args = parser.parse_args(argv) + try: + receipt = asyncio.run( + measure_query_envelope( + args.archive_root, + expression=args.expression, + rounds=args.rounds, + warmup=args.warmup, + baseline_rounds=args.baseline_rounds, + tolerance=args.tolerance, + sample_interval_s=args.sample_interval, + max_rss_bytes=args.max_rss_mb * MIB, + max_pss_bytes=args.max_pss_mb * MIB, + max_swap_growth_bytes=args.max_swap_growth_mb * MIB, + max_temp_growth_bytes=args.max_temp_growth_mb * MIB, + ) + ) + except (SchemaVersionMismatchError, ResourceProbeUnavailableError) as exc: + regression_path = ( + "Re-run against the exact promoted generation after its schema lifecycle action completes; " + "do not bypass the archive compatibility check." + if isinstance(exc, SchemaVersionMismatchError) + else "Re-run on a host whose procfs reports VmRSS, Pss, and VmSwap; the envelope is unmeasured here." + ) + receipt = { + "status": "blocked-env", + "archive_root": str(args.archive_root.resolve()), + "blocking_error": str(exc), + "regression_path": regression_path, + } + text = json.dumps(receipt, indent=2, sort_keys=True) + print(text) + if args.receipt: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(text + "\n", encoding="utf-8") + return 2 + except (FileNotFoundError, ValueError) as exc: + parser.error(str(exc)) + text = json.dumps(receipt, indent=2, sort_keys=True) + print(text) + if args.receipt: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(text + "\n", encoding="utf-8") + return 0 if receipt["returned_to_envelope"] else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/devtools.md b/docs/devtools.md index 570b9fa468..0a6459cdcc 100644 --- a/docs/devtools.md +++ b/docs/devtools.md @@ -97,6 +97,7 @@ These are the commands worth remembering during normal repo work: | --- | --- | | `devtools bench memory` | Measure query-memory envelopes on generated fixtures. | | `devtools bench pipeline` | Run typed pipeline probes against synthetic, staged, or archive-subset inputs. | +| `devtools bench query-envelope` | Measure repeated incident-scale query RSS, PSS, swap, and temp envelopes. | | `devtools bench slo` | Check read-surface latency budgets in docs/plans/slo-catalog.yaml against benchmark measurements. | ### Archive diff --git a/docs/maintenance.md b/docs/maintenance.md index 7f9eecbb7c..099fd34e87 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -295,6 +295,44 @@ recorded separately in the receipt so a retry can safely continue an additive repair. Reindex acceptance runs the same closure check against the candidate index before promotion. +### `polylogue ops maintenance blob-disposition` — physical namespace disposition + +One-time transition tooling for the blob-store maneuver. `plan` is read-only: +it walks the complete physical namespace and gives every object exactly one +disposition proven against a configured source — `source_present`, +`superseded_prefix`, `restore_required`, or `unresolved`. A plan is acceptable +only at zero unresolved members, and its digest binds the archive identity, +the namespace, the denominators, and every member outcome. + +```bash +polylogue ops maintenance blob-disposition plan \ + --archive-root /path/to/archive \ + --output /path/to/disposition-plan.json --output-format json +polylogue ops maintenance blob-disposition apply \ + --archive-root /path/to/archive \ + --plan /path/to/disposition-plan.json \ + --authorized-digest \ + --receipt /path/to/new/disposition-receipt.json --active +``` + +`restore` is the additive half on its own: it publishes sole-copy carriers +into their ordinary spool and deletes nothing, so it does not wait on the +plan reaching zero unresolved. `apply` is a dry rehearsal without `--active`. It makes no classification +judgment: it revalidates every member's own proof immediately before its +effect, restores sole-copy carriers into their ordinary spool before any +deletion, never touches a historical carrier during restoration, and deletes +only unreferenced members through the canonical blob-GC seam. Any drift — a +changed source, a changed object, a new referent, a different digest or +denominator — refuses the whole plan. + +Hook-event and browser-capture carriers are proven by the owning production +read route, not by bytes: acquisition derives fields the spool file does not +carry, so byte equality would misreport reproducible material as a sole copy. + +Deletion trigger: this command, both maintenance modules, and their tests are +removed with the terminal disposition receipt. The recurring liveness, +publication, GC, and spool-admission laws stay with their owners. + ### `polylogue ops maintenance preview` — staleness inventory Read-only. Produces a per-model inventory of stale, missing, orphan, diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 79b0fee903..17e77c795c 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -219,9 +219,15 @@ "blob_residue_compare_command", "Compare present blob-residue candidates through the production parse route.", ), + ( + "blob-disposition", + "_blob_disposition", + "blob_disposition_group", + "Compile or consume the physical blob namespace disposition plan.", + ), ) -_NESTED_GROUP_COMMANDS = frozenset({"archive-root-relocation", "source-continuity-recovery"}) +_NESTED_GROUP_COMMANDS = frozenset({"archive-root-relocation", "blob-disposition", "source-continuity-recovery"}) @click.group("maintenance") diff --git a/polylogue/cli/commands/maintenance/_blob_disposition.py b/polylogue/cli/commands/maintenance/_blob_disposition.py new file mode 100644 index 0000000000..e3297b3df7 --- /dev/null +++ b/polylogue/cli/commands/maintenance/_blob_disposition.py @@ -0,0 +1,254 @@ +"""``maintenance blob-disposition``: compile or consume one disposition plan.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import click + + +@click.group("blob-disposition") +def blob_disposition_group() -> None: + """Plan and consume the physical blob namespace disposition.""" + + +@blob_disposition_group.command("plan") +@click.option( + "--archive-root", + type=click.Path(path_type=Path, exists=True, file_okay=False, readable=True), + required=True, + help="Archive root whose blob namespace is planned.", +) +@click.option( + "--output", + type=click.Path(path_type=Path, dir_okay=False), + required=True, + help="Destination for the immutable plan artifact.", +) +@click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) +def blob_disposition_plan_command(archive_root: Path, output: Path, output_format: str) -> None: + """Compile a read-only, zero-unknown disposition plan. Never mutates.""" + from polylogue.maintenance.blob_disposition import compile_disposition_plan, resolve_disposition_roots + + _, hook_sources, capture_spool = resolve_disposition_roots(archive_root) + try: + plan = compile_disposition_plan( + archive_root=archive_root, + blob_root=archive_root / "blob", + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(plan.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + "\n") + except (OSError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + summary = { + "plan": str(output), + "digest": plan.digest(), + "accepted": plan.accepted, + "counts": plan.counts, + "bytes_by_disposition": plan.bytes_by_disposition, + "denominator": plan.denominator.to_dict(), + } + if output_format == "json": + click.echo(json.dumps(summary, sort_keys=True)) + return + click.echo(f"Disposition plan: {output}") + click.echo(f"Digest: {plan.digest()}") + click.echo(f"Accepted (zero unresolved): {plan.accepted}") + click.echo(f"Counts: {json.dumps(plan.counts, sort_keys=True)}") + click.echo("Read-only: true") + + +@blob_disposition_group.command("restore") +@click.option( + "--archive-root", + type=click.Path(path_type=Path, exists=True, file_okay=False, readable=True), + required=True, + help="Archive root the plan was compiled from.", +) +@click.option( + "--plan", + "plan_path", + type=click.Path(path_type=Path, exists=True, dir_okay=False, readable=True), + required=True, + help="The plan naming the sole-copy carriers to restore.", +) +@click.option("--authorized-digest", required=True, help="Digest of the reviewed plan.") +@click.option( + "--receipt", + type=click.Path(path_type=Path, dir_okay=False), + required=True, + help="Destination for the restoration receipt.", +) +@click.option( + "--active", + is_flag=True, + default=False, + help="Perform the restorations. Without it the run is a dry rehearsal.", +) +@click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) +def blob_disposition_restore_command( + archive_root: Path, + plan_path: Path, + authorized_digest: str, + receipt: Path, + active: bool, + output_format: str, +) -> None: + """Restore sole-copy carriers into their ordinary spool. Deletes nothing. + + Restoration is additive, so it does not wait on the whole plan reaching + zero unresolved: withholding it would leave the only carrier of wanted + material unpreserved while unrelated objects are still being classified. + """ + from polylogue.maintenance.blob_disposition import ( + BlobDispositionPlan, + build_disposition_context, + resolve_disposition_roots, + ) + from polylogue.maintenance.blob_disposition_apply import ( + TOOL_VERSION, + DispositionApplyReceipt, + restore_plan_members, + write_receipt, + ) + + hooks_root, hook_sources, capture_spool = resolve_disposition_roots(archive_root) + try: + plan = BlobDispositionPlan.from_dict(json.loads(plan_path.read_text(encoding="utf-8"))) + if plan.digest() != authorized_digest: + raise click.ClickException("authorized digest does not match the plan") + context = build_disposition_context( + archive_root=archive_root, + blob_root=archive_root / "blob", + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + results = restore_plan_members( + plan, + context=context, + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=not active, + ) + result = DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=not active, + results=results, + ) + write_receipt(receipt, result) + except (OSError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + if output_format == "json": + click.echo(json.dumps({"receipt": str(receipt), "ok": result.ok, "counts": result.counts}, sort_keys=True)) + else: + click.echo(f"Restoration receipt: {receipt}") + click.echo(f"Dry run: {not active}") + click.echo(f"Counts: {json.dumps(result.counts, sort_keys=True)}") + if not result.ok: + raise SystemExit(1) + + +@blob_disposition_group.command("apply") +@click.option( + "--archive-root", + type=click.Path(path_type=Path, exists=True, file_okay=False, readable=True), + required=True, + help="Archive root the authorized plan was compiled from.", +) +@click.option( + "--plan", + "plan_path", + type=click.Path(path_type=Path, exists=True, dir_okay=False, readable=True), + required=True, + help="The exact accepted plan artifact.", +) +@click.option("--authorized-digest", required=True, help="Digest of the independently accepted plan.") +@click.option( + "--receipt", + type=click.Path(path_type=Path, dir_okay=False), + required=True, + help="Destination for the complete before/after receipt.", +) +@click.option( + "--active", + is_flag=True, + default=False, + help="Perform the authorized effects. Without it the run is a dry rehearsal.", +) +@click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) +def blob_disposition_apply_command( + archive_root: Path, + plan_path: Path, + authorized_digest: str, + receipt: Path, + active: bool, + output_format: str, +) -> None: + """Restore sole copies, then delete proven-redundant objects.""" + from polylogue.config import Config + from polylogue.maintenance.blob_disposition import ( + BlobDispositionPlan, + build_disposition_context, + resolve_disposition_roots, + ) + from polylogue.maintenance.blob_disposition_apply import apply_disposition_plan, write_receipt + from polylogue.maintenance.offline_guard import offline_writer_block_reason + from polylogue.paths import render_root + + hooks_root, hook_sources, capture_spool = resolve_disposition_roots(archive_root) + try: + plan = BlobDispositionPlan.from_dict(json.loads(plan_path.read_text(encoding="utf-8"))) + context = build_disposition_context( + archive_root=archive_root, + blob_root=archive_root / "blob", + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + block_reason = offline_writer_block_reason( + Config(archive_root=archive_root, render_root=render_root(), sources=[]) + ) + result = apply_disposition_plan( + plan, + context=context, + authorized_digest=authorized_digest, + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + writer_block_reason=block_reason, + dry_run=not active, + ) + write_receipt(receipt, result) + except (OSError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + summary = {"receipt": str(receipt), "ok": result.ok, "counts": result.counts, "blockers": list(result.blockers)} + if output_format == "json": + click.echo(json.dumps(summary, sort_keys=True)) + else: + click.echo(f"Disposition receipt: {receipt}") + click.echo(f"Dry run: {not active}") + click.echo(f"Counts: {json.dumps(result.counts, sort_keys=True)}") + for blocker in result.blockers: + click.echo(f"Blocked: {blocker}") + if not result.ok: + raise SystemExit(1) + + +__all__ = [ + "blob_disposition_apply_command", + "blob_disposition_group", + "blob_disposition_plan_command", + "blob_disposition_restore_command", +] diff --git a/polylogue/maintenance/blob_disposition.py b/polylogue/maintenance/blob_disposition.py new file mode 100644 index 0000000000..446ba22a6f --- /dev/null +++ b/polylogue/maintenance/blob_disposition.py @@ -0,0 +1,849 @@ +"""Read-only disposition plan for the physical blob namespace. + +The blob store is forensic evidence, never desired-state authority. Every +physical object therefore receives exactly one disposition proven against a +configured source: + +``source_present`` + Current source material reproduces the object's content, byte-identically + or through the owning production route's semantic equality. The object is + redundant storage and may be removed. +``superseded_prefix`` + The object is the exact prefix of a larger retained carrier of the same + logical source item (append lineage). It may be removed. +``restore_required`` + The object is the only verified carrier of wanted material and names an + ordinary spool destination that current acquisition admits. Restoration + precedes any removal. +``unresolved`` + Nothing above holds. Unresolved blocks: it is never downgraded to + discard, and it never authorizes restoration. + +A plan is acceptable only at zero unresolved members. It is immutable, bound +to the archive identity, blob namespace identity, and exact denominators it +was compiled from, and consumed by :mod:`polylogue.maintenance. +blob_disposition_apply` under a separate authorization. + +This is a one-time transition planner. Its deletion trigger is the terminal +disposition receipt: once the physical namespace is accounted for, this +module and its apply sibling go with it, and only the recurring liveness, +publication, GC, and spool-admission laws remain in their owners. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sqlite3 +from collections.abc import Mapping, Sequence +from contextlib import closing +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from typing import IO, Protocol + +from polylogue.storage.blob_store import BlobNamespaceEntry, BlobNamespaceEntryKind, BlobStore + +TOOL_VERSION = "blob-disposition-plan-v1" + +_HASH_CHUNK_BYTES = 1 << 20 +# An object larger than this is never one of the small JSON envelopes the +# spool provers own; probing it would read hundreds of megabytes to decide a +# question its size already answers. +_MAX_ENVELOPE_PROBE_BYTES = 256 << 20 + + +class BlobDispositionError(RuntimeError): + """Raised when a disposition plan cannot be compiled or trusted.""" + + +class BlobDisposition(StrEnum): + """The only terminal dispositions a physical blob may receive.""" + + SOURCE_PRESENT = "source_present" + SUPERSEDED_PREFIX = "superseded_prefix" + RESTORE_REQUIRED = "restore_required" + UNRESOLVED = "unresolved" + + +class SourceProofMode(StrEnum): + """How a prover established that current source material holds the content.""" + + BYTE_IDENTICAL = "byte_identical" + SEMANTIC_EQUIVALENT = "semantic_equivalent" + STRICT_PREFIX = "strict_prefix" + + +class RestorationDestination(StrEnum): + """Ordinary spool destinations current acquisition already admits.""" + + HOOK_EVENT_SPOOL = "hook_event_spool" + BROWSER_CAPTURE_SPOOL = "browser_capture_spool" + + +@dataclass(frozen=True, slots=True) +class SourceProof: + """One prover's evidence that a configured source holds the content.""" + + prover: str + mode: SourceProofMode + source_id: str + source_path: str + detail: str = "" + + def to_dict(self) -> dict[str, str]: + return { + "prover": self.prover, + "mode": self.mode.value, + "source_id": self.source_id, + "source_path": self.source_path, + "detail": self.detail, + } + + +@dataclass(frozen=True, slots=True) +class RestorationTarget: + """Where a sole-copy carrier is restored before its removal is considered.""" + + destination: RestorationDestination + logical_id: str + + def to_dict(self) -> dict[str, str]: + return {"destination": self.destination.value, "logical_id": self.logical_id} + + +@dataclass(frozen=True, slots=True) +class BlobDispositionMember: + """One physical blob and its single proven disposition.""" + + blob_hash: str + size_bytes: int + referenced: bool + disposition: BlobDisposition + reason: str + proof: SourceProof | None = None + restoration: RestorationTarget | None = None + + def to_dict(self) -> dict[str, object]: + return { + "blob_hash": self.blob_hash, + "size_bytes": self.size_bytes, + "referenced": self.referenced, + "disposition": self.disposition.value, + "reason": self.reason, + "proof": self.proof.to_dict() if self.proof is not None else None, + "restoration": self.restoration.to_dict() if self.restoration is not None else None, + } + + +@dataclass(frozen=True, slots=True) +class BlobDispositionDenominator: + """The exact population a plan was compiled from.""" + + physical_file_count: int + distinct_hash_count: int + total_bytes: int + referenced_hash_count: int + referenced_present_count: int + referenced_absent_count: int + invalid_namespace_entries: tuple[str, ...] = () + + def to_dict(self) -> dict[str, object]: + return { + "physical_file_count": self.physical_file_count, + "distinct_hash_count": self.distinct_hash_count, + "total_bytes": self.total_bytes, + "referenced_hash_count": self.referenced_hash_count, + "referenced_present_count": self.referenced_present_count, + "referenced_absent_count": self.referenced_absent_count, + "invalid_namespace_entries": list(self.invalid_namespace_entries), + } + + +@dataclass(frozen=True, slots=True) +class BlobDispositionPlan: + """An immutable, identity-bound, zero-unknown disposition plan.""" + + tool_version: str + archive_root: str + blob_root: str + denominator: BlobDispositionDenominator + members: tuple[BlobDispositionMember, ...] + + @property + def counts(self) -> dict[str, int]: + counts = {disposition.value: 0 for disposition in BlobDisposition} + for member in self.members: + counts[member.disposition.value] += 1 + return counts + + @property + def bytes_by_disposition(self) -> dict[str, int]: + totals = {disposition.value: 0 for disposition in BlobDisposition} + for member in self.members: + totals[member.disposition.value] += member.size_bytes + return totals + + @property + def unresolved_count(self) -> int: + return self.counts[BlobDisposition.UNRESOLVED.value] + + @property + def accepted(self) -> bool: + """A plan is acceptable only when nothing is unexplained.""" + return self.unresolved_count == 0 and not self.denominator.invalid_namespace_entries + + def members_for(self, disposition: BlobDisposition) -> tuple[BlobDispositionMember, ...]: + return tuple(member for member in self.members if member.disposition is disposition) + + def to_dict(self) -> dict[str, object]: + return { + "tool_version": self.tool_version, + "archive_root": self.archive_root, + "blob_root": self.blob_root, + "denominator": self.denominator.to_dict(), + "counts": self.counts, + "bytes_by_disposition": self.bytes_by_disposition, + "unresolved_count": self.unresolved_count, + "accepted": self.accepted, + "read_only": True, + "members": [member.to_dict() for member in self.members], + } + + def digest(self) -> str: + """Bind identity, denominators, and every exact member outcome.""" + payload = { + "tool_version": self.tool_version, + "archive_root": self.archive_root, + "blob_root": self.blob_root, + "denominator": self.denominator.to_dict(), + "members": [member.to_dict() for member in self.members], + } + canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + @classmethod + def from_dict(cls, payload: Mapping[str, object]) -> BlobDispositionPlan: + """Reload a persisted plan without re-deriving any judgment.""" + try: + denominator = payload["denominator"] + raw_members = payload["members"] + if not isinstance(denominator, Mapping) or not isinstance(raw_members, list): + raise BlobDispositionError("plan denominator and members must be structured") + members = tuple(_member_from_dict(item) for item in raw_members) + return cls( + tool_version=str(payload["tool_version"]), + archive_root=str(payload["archive_root"]), + blob_root=str(payload["blob_root"]), + denominator=BlobDispositionDenominator( + physical_file_count=int(denominator["physical_file_count"]), + distinct_hash_count=int(denominator["distinct_hash_count"]), + total_bytes=int(denominator["total_bytes"]), + referenced_hash_count=int(denominator["referenced_hash_count"]), + referenced_present_count=int(denominator["referenced_present_count"]), + referenced_absent_count=int(denominator["referenced_absent_count"]), + invalid_namespace_entries=tuple( + str(entry) for entry in denominator.get("invalid_namespace_entries", ()) + ), + ), + members=members, + ) + except (KeyError, TypeError, ValueError) as exc: + raise BlobDispositionError(f"unreadable disposition plan: {exc}") from exc + + +def _member_from_dict(payload: object) -> BlobDispositionMember: + if not isinstance(payload, Mapping): + raise BlobDispositionError("plan member must be an object") + proof_payload = payload.get("proof") + proof = None + if isinstance(proof_payload, Mapping): + proof = SourceProof( + prover=str(proof_payload["prover"]), + mode=SourceProofMode(str(proof_payload["mode"])), + source_id=str(proof_payload["source_id"]), + source_path=str(proof_payload["source_path"]), + detail=str(proof_payload.get("detail", "")), + ) + restoration_payload = payload.get("restoration") + restoration = None + if isinstance(restoration_payload, Mapping): + restoration = RestorationTarget( + destination=RestorationDestination(str(restoration_payload["destination"])), + logical_id=str(restoration_payload["logical_id"]), + ) + return BlobDispositionMember( + blob_hash=str(payload["blob_hash"]), + size_bytes=int(payload["size_bytes"]), + referenced=bool(payload["referenced"]), + disposition=BlobDisposition(str(payload["disposition"])), + reason=str(payload["reason"]), + proof=proof, + restoration=restoration, + ) + + +class BlobSourceProver(Protocol): + """Establishes that configured source material still holds a blob's content.""" + + name: str + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: ... + + +class BlobRestorationResolver(Protocol): + """Names the ordinary spool destination a sole-copy carrier belongs to.""" + + def restoration_target(self, path: Path) -> RestorationTarget | None: ... + + +def _read_envelope(path: Path, *, expected_keys: frozenset[str]) -> dict[str, object] | None: + """Load a small JSON object envelope, refusing anything of another shape.""" + try: + if path.stat().st_size > _MAX_ENVELOPE_PROBE_BYTES: + return None + with path.open("rb") as handle: + head = handle.read(1) + if head != b"{": + return None + handle.seek(0) + value = json.load(handle) + except (OSError, json.JSONDecodeError, RecursionError, ValueError): + return None + if not isinstance(value, dict) or not expected_keys.issubset(value): + return None + return value + + +class HookEventSpoolProver: + """Prove a hook-event envelope against the declared hook spool topology. + + Acquisition stores the *validated* record, whose ``observed_at_ms`` the + spool file does not carry, and both sides are serialized independently. + Byte equality is therefore the wrong law here: the proof is equality of + the production-route record, which is what admission would reproduce. + """ + + name = "hook-event-spool" + _ENVELOPE_KEYS = frozenset({"event_id", "event_type", "session_id", "timestamp", "provider", "payload"}) + + def __init__(self, sources: Sequence[tuple[str, Path]]) -> None: + self._sources = tuple(sources) + self._index: dict[str, tuple[str, Path]] | None = None + + def _spool_index(self) -> dict[str, tuple[str, Path]]: + if self._index is not None: + return self._index + index: dict[str, tuple[str, Path]] = {} + for source_id, root in self._sources: + for directory, subdirectories, filenames in os.walk(root): + subdirectories.sort() + for filename in sorted(filenames): + if not filename.endswith(".json"): + continue + index.setdefault(filename[: -len(".json")], (source_id, Path(directory) / filename)) + self._index = index + return index + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + envelope = _read_envelope(path, expected_keys=self._ENVELOPE_KEYS) + if envelope is None: + return None + event_id = envelope.get("event_id") + if not isinstance(event_id, str) or not event_id: + return None + located = self._spool_index().get(event_id) + if located is None: + return None + source_id, spool_path = located + from polylogue.sources.hooks import HookSpoolRecordError, read_hook_spool_record + + try: + record = read_hook_spool_record(spool_path) + except HookSpoolRecordError: + return None + if record != envelope: + return None + return SourceProof( + prover=self.name, + mode=SourceProofMode.SEMANTIC_EQUIVALENT, + source_id=source_id, + source_path=str(spool_path), + detail=f"hook event {event_id} reproduces through the spool read route", + ) + + def restoration_target(self, path: Path) -> RestorationTarget | None: + envelope = _read_envelope(path, expected_keys=self._ENVELOPE_KEYS) + if envelope is None: + return None + event_id = envelope.get("event_id") + if not isinstance(event_id, str) or not event_id: + return None + return RestorationTarget(destination=RestorationDestination.HOOK_EVENT_SPOOL, logical_id=event_id) + + +class BrowserCaptureSpoolProver: + """Prove a browser-capture envelope against the ordinary capture spool.""" + + name = "browser-capture-spool" + _ENVELOPE_KEYS = frozenset({"polylogue_capture_kind", "schema_version", "session", "provenance"}) + + def __init__(self, spool_root: Path, *, source_id: str = "browser-capture-spool") -> None: + self._spool_root = spool_root + self._source_id = source_id + + def _envelope(self, path: Path) -> object | None: + payload = _read_envelope(path, expected_keys=self._ENVELOPE_KEYS) + if payload is None: + return None + from pydantic import ValidationError + + from polylogue.browser_capture.models import BrowserCaptureEnvelope + + try: + return BrowserCaptureEnvelope.model_validate(payload) + except ValidationError: + return None + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + envelope = self._envelope(path) + if envelope is None: + return None + from polylogue.browser_capture.models import BrowserCaptureEnvelope + from polylogue.browser_capture.receiver import capture_artifact_path, capture_dedup_content_hash + + assert isinstance(envelope, BrowserCaptureEnvelope) + spooled = capture_artifact_path(envelope, self._spool_root) + if not spooled.is_file(): + return None + try: + existing = BrowserCaptureEnvelope.model_validate_json(spooled.read_bytes()) + except (OSError, ValueError): + return None + if capture_dedup_content_hash(existing) != capture_dedup_content_hash(envelope): + return None + return SourceProof( + prover=self.name, + mode=SourceProofMode.SEMANTIC_EQUIVALENT, + source_id=self._source_id, + source_path=str(spooled), + detail="capture spool holds a dedup-equivalent envelope", + ) + + def restoration_target(self, path: Path) -> RestorationTarget | None: + envelope = self._envelope(path) + if envelope is None: + return None + from polylogue.browser_capture.models import BrowserCaptureEnvelope + + assert isinstance(envelope, BrowserCaptureEnvelope) + return RestorationTarget( + destination=RestorationDestination.BROWSER_CAPTURE_SPOOL, + logical_id=f"{envelope.session.provider}:{envelope.session.provider_session_id}", + ) + + +def _hash_stream(handle: IO[bytes], *, limit: int | None = None) -> tuple[str, int]: + digest = hashlib.sha256() + consumed = 0 + while True: + want = _HASH_CHUNK_BYTES if limit is None else min(_HASH_CHUNK_BYTES, limit - consumed) + if want <= 0: + break + chunk = handle.read(want) + if not chunk: + break + digest.update(chunk) + consumed += len(chunk) + return digest.hexdigest(), consumed + + +@dataclass(frozen=True, slots=True) +class RawSourceCarrier: + """One acquisition's record of where a payload came from.""" + + source_path: str + append_start_offset: int | None = None + + +class RawSourceFileProver: + """Prove a raw payload against the source file it was acquired from. + + Three shapes all reproduce the content and all require a fresh hash: + the whole file, the file's own prefix (append-structured providers grow + in place), and the recorded append span for a row that captured only its + own increment. Path existence proves nothing. + """ + + name = "raw-source-file" + + def __init__(self, carriers_by_hash: Mapping[str, tuple[RawSourceCarrier, ...]]) -> None: + self._carriers = dict(carriers_by_hash) + + def _attempt(self, source: Path, *, offset: int, size_bytes: int, whole: bool) -> tuple[str, int] | None: + try: + with source.open("rb") as handle: + if offset: + handle.seek(offset) + return _hash_stream(handle, limit=None if whole else size_bytes) + except OSError: + return None + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + for carrier in self._carriers.get(blob_hash, ()): + source = Path(carrier.source_path) + try: + if not source.is_file(): + continue + source_size = source.stat().st_size + except OSError: + continue + attempts: list[tuple[SourceProofMode, int, bool]] = [] + if source_size == size_bytes: + attempts.append((SourceProofMode.BYTE_IDENTICAL, 0, True)) + elif source_size > size_bytes: + attempts.append((SourceProofMode.STRICT_PREFIX, 0, False)) + offset = carrier.append_start_offset + if offset is not None and offset > 0 and source_size >= offset + size_bytes: + attempts.append((SourceProofMode.STRICT_PREFIX, offset, False)) + for mode, start, whole in attempts: + measured = self._attempt(source, offset=start, size_bytes=size_bytes, whole=whole) + if measured is None: + continue + digest, consumed = measured + if consumed != size_bytes or digest != blob_hash: + continue + span = "whole file" if whole else f"{size_bytes} bytes at offset {start}" + return SourceProof( + prover=self.name, + mode=mode, + source_id="configured-source-file", + source_path=str(source), + detail=f"fresh hash over the {span} of the live source", + ) + return None + + +class AppendPrefixProver: + """Prove a blob is the exact prefix of a retained carrier of the same item. + + Scoped to carriers that share a logical source identity: an unrelated + object that merely happens to start with the same bytes is not append + lineage, and treating it as such would discard a distinct carrier. + """ + + name = "append-prefix" + + def __init__(self, successors_by_hash: Mapping[str, tuple[str, ...]], *, blob_store: BlobStore) -> None: + self._successors = dict(successors_by_hash) + self._store = blob_store + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + for successor in self._successors.get(blob_hash, ()): + successor_path = self._store.blob_path(successor) + try: + if not successor_path.is_file() or successor_path.stat().st_size <= size_bytes: + continue + with successor_path.open("rb") as handle: + digest, consumed = _hash_stream(handle, limit=size_bytes) + except OSError: + continue + if consumed != size_bytes or digest != blob_hash: + continue + return SourceProof( + prover=self.name, + mode=SourceProofMode.STRICT_PREFIX, + source_id="retained-blob", + source_path=successor, + detail="exact prefix of a larger retained carrier of the same logical item", + ) + return None + + +@dataclass(frozen=True, slots=True) +class BlobDispositionContext: + """Everything a compilation needs, resolved once and reused per member.""" + + blob_store: BlobStore + provers: tuple[BlobSourceProver, ...] + referenced_hashes: frozenset[str] + restoration_provers: tuple[BlobRestorationResolver, ...] = field(default=()) + + +def _open_ro(path: Path) -> sqlite3.Connection: + return sqlite3.connect(f"file:{path}?mode=ro", uri=True) + + +def referenced_blob_hashes(source_db: Path) -> frozenset[str]: + """Union every durable relation that names a physical blob hash. + + A relation that exists but cannot be read is a failure, never an empty + set: reading zero references from an unreadable tier would license + deleting the whole namespace. + """ + relations = ( + ("blob_refs", "blob_hash"), + ("raw_sessions", "blob_hash"), + ("raw_hook_events", "blob_hash"), + ("raw_artifacts", "blob_hash"), + ("blob_publication_reservations", "blob_hash"), + ) + hashes: set[str] = set() + with closing(_open_ro(source_db)) as conn: + present = { + str(row[0]) + for row in conn.execute("SELECT name FROM sqlite_master WHERE type IN ('table','view')").fetchall() + } + for table, column in relations: + if table not in present: + continue + try: + columns = {str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})").fetchall()} + except sqlite3.Error as exc: + raise BlobDispositionError(f"reference relation {table} is unreadable: {exc}") from exc + if column not in columns: + continue + try: + rows = conn.execute( + f"SELECT DISTINCT lower(hex({column})) FROM {table} WHERE {column} IS NOT NULL" + ).fetchall() + except sqlite3.Error as exc: + raise BlobDispositionError(f"reference relation {table} is unreadable: {exc}") from exc + hashes.update(str(row[0]) for row in rows) + return frozenset(hashes) + + +def raw_source_carriers_by_hash(source_db: Path) -> dict[str, tuple[RawSourceCarrier, ...]]: + """Map each acquired payload hash to the source carriers that produced it.""" + mapping: dict[str, set[RawSourceCarrier]] = {} + with closing(_open_ro(source_db)) as conn: + try: + rows = conn.execute( + "SELECT lower(hex(blob_hash)), source_path, append_start_offset FROM raw_sessions " + "WHERE blob_hash IS NOT NULL AND source_path IS NOT NULL" + ).fetchall() + except sqlite3.Error as exc: + raise BlobDispositionError(f"raw_sessions is unreadable: {exc}") from exc + for blob_hash, source_path, offset in rows: + carrier = RawSourceCarrier(str(source_path), int(offset) if offset is not None else None) + mapping.setdefault(str(blob_hash), set()).add(carrier) + return { + key: tuple(sorted(value, key=lambda item: (item.source_path, item.append_start_offset or 0))) + for key, value in mapping.items() + } + + +def append_successors_by_hash(source_db: Path) -> dict[str, tuple[str, ...]]: + """Map each payload hash to larger carriers of the same logical item.""" + with closing(_open_ro(source_db)) as conn: + try: + rows = conn.execute( + "SELECT origin, native_id, lower(hex(blob_hash)), blob_size FROM raw_sessions " + "WHERE blob_hash IS NOT NULL AND native_id IS NOT NULL" + ).fetchall() + except sqlite3.Error as exc: + raise BlobDispositionError(f"raw_sessions is unreadable: {exc}") from exc + grouped: dict[tuple[str, str], list[tuple[int, str]]] = {} + for origin, native_id, blob_hash, size in rows: + if size is None: + continue + grouped.setdefault((str(origin), str(native_id)), []).append((int(size), str(blob_hash))) + successors: dict[str, tuple[str, ...]] = {} + for carriers in grouped.values(): + carriers.sort() + for index, (size, blob_hash) in enumerate(carriers): + larger = tuple(other for other_size, other in carriers[index + 1 :] if other_size > size) + if larger: + successors[blob_hash] = larger + return successors + + +def _restoration_target(path: Path, provers: Sequence[BlobRestorationResolver]) -> RestorationTarget | None: + for prover in provers: + target = prover.restoration_target(path) + if target is not None: + return target + return None + + +def classify_blob( + entry: BlobNamespaceEntry, + *, + context: BlobDispositionContext, +) -> BlobDispositionMember: + """Assign exactly one disposition to one physical blob.""" + assert entry.hash_hex is not None + blob_hash = entry.hash_hex + try: + size_bytes = entry.path.stat().st_size + except OSError as exc: + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=0, + referenced=blob_hash in context.referenced_hashes, + disposition=BlobDisposition.UNRESOLVED, + reason=f"physical object is unreadable: {exc}", + ) + referenced = blob_hash in context.referenced_hashes + for prover in context.provers: + proof = prover.prove(blob_hash, entry.path, size_bytes) + if proof is None: + continue + disposition = ( + BlobDisposition.SUPERSEDED_PREFIX + if proof.prover == AppendPrefixProver.name + else BlobDisposition.SOURCE_PRESENT + ) + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=size_bytes, + referenced=referenced, + disposition=disposition, + reason=f"{proof.prover} proved {proof.mode.value}", + proof=proof, + ) + restoration = _restoration_target(entry.path, context.restoration_provers) + if restoration is not None: + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=size_bytes, + referenced=referenced, + disposition=BlobDisposition.RESTORE_REQUIRED, + reason="no configured source holds this content and it names an ordinary spool destination", + restoration=restoration, + ) + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=size_bytes, + referenced=referenced, + disposition=BlobDisposition.UNRESOLVED, + reason="no source proof and no ordinary restoration destination", + ) + + +def resolve_disposition_roots(archive_root: Path) -> tuple[Path, tuple[tuple[str, Path], ...], Path]: + """Resolve the primary hook spool, the declared spool topology, and captures. + + The declared topology already includes the legacy read-only roots, so a + carrier whose event still sits in a superseded spool is proven at a + configured source rather than restored a second time. + """ + from polylogue.sources.hooks import hook_spool_sources + + hooks_root = archive_root / "hooks" + sources = tuple( + (spec.source_id, spec.root) for spec in hook_spool_sources(primary_root=hooks_root) if spec.root.is_dir() + ) + return hooks_root, sources, archive_root / "browser-capture" + + +def build_disposition_context( + *, + archive_root: Path, + blob_root: Path, + source_db: Path, + hook_spool_sources: Sequence[tuple[str, Path]], + browser_capture_spool: Path, +) -> BlobDispositionContext: + """Resolve the prover set from configured sources, not from history.""" + store = BlobStore(blob_root) + hook_prover = HookEventSpoolProver(hook_spool_sources) + capture_prover = BrowserCaptureSpoolProver(browser_capture_spool) + provers: tuple[BlobSourceProver, ...] = ( + hook_prover, + capture_prover, + RawSourceFileProver(raw_source_carriers_by_hash(source_db)), + AppendPrefixProver(append_successors_by_hash(source_db), blob_store=store), + ) + return BlobDispositionContext( + blob_store=store, + provers=provers, + referenced_hashes=referenced_blob_hashes(source_db), + restoration_provers=(hook_prover, capture_prover), + ) + + +def compile_disposition_plan( + *, + archive_root: Path, + blob_root: Path, + source_db: Path, + context: BlobDispositionContext | None = None, + hook_spool_sources: Sequence[tuple[str, Path]] | None = None, + browser_capture_spool: Path | None = None, + progress: object | None = None, +) -> BlobDispositionPlan: + """Walk the complete physical namespace and compile one immutable plan.""" + if context is None: + if hook_spool_sources is None or browser_capture_spool is None: + raise BlobDispositionError("compilation needs either a context or the configured spool roots") + context = build_disposition_context( + archive_root=archive_root, + blob_root=blob_root, + source_db=source_db, + hook_spool_sources=hook_spool_sources, + browser_capture_spool=browser_capture_spool, + ) + members: list[BlobDispositionMember] = [] + invalid: list[str] = [] + seen: set[str] = set() + file_count = 0 + for entry in context.blob_store.iter_namespace(): + if entry.kind is not BlobNamespaceEntryKind.BLOB: + invalid.append(f"{entry.relative_path}: {entry.issue.value if entry.issue else 'unclassified'}") + continue + file_count += 1 + assert entry.hash_hex is not None + if entry.hash_hex in seen: + continue + seen.add(entry.hash_hex) + members.append(classify_blob(entry, context=context)) + if progress is not None and len(members) % 1000 == 0: + progress(len(members)) # type: ignore[operator] + present = frozenset(seen) + denominator = BlobDispositionDenominator( + physical_file_count=file_count, + distinct_hash_count=len(members), + total_bytes=sum(member.size_bytes for member in members), + referenced_hash_count=len(context.referenced_hashes), + referenced_present_count=len(context.referenced_hashes & present), + referenced_absent_count=len(context.referenced_hashes - present), + invalid_namespace_entries=tuple(sorted(invalid)), + ) + return BlobDispositionPlan( + tool_version=TOOL_VERSION, + archive_root=str(archive_root), + blob_root=str(blob_root), + denominator=denominator, + members=tuple(sorted(members, key=lambda member: member.blob_hash)), + ) + + +__all__ = [ + "TOOL_VERSION", + "AppendPrefixProver", + "BlobDisposition", + "BlobDispositionContext", + "BlobDispositionDenominator", + "BlobDispositionError", + "BlobDispositionMember", + "BlobDispositionPlan", + "BlobRestorationResolver", + "BlobSourceProver", + "BrowserCaptureSpoolProver", + "HookEventSpoolProver", + "RawSourceCarrier", + "RawSourceFileProver", + "RestorationDestination", + "RestorationTarget", + "SourceProof", + "SourceProofMode", + "append_successors_by_hash", + "build_disposition_context", + "classify_blob", + "compile_disposition_plan", + "raw_source_carriers_by_hash", + "resolve_disposition_roots", + "referenced_blob_hashes", +] diff --git a/polylogue/maintenance/blob_disposition_apply.py b/polylogue/maintenance/blob_disposition_apply.py new file mode 100644 index 0000000000..e0b70a06fe --- /dev/null +++ b/polylogue/maintenance/blob_disposition_apply.py @@ -0,0 +1,475 @@ +"""Consume one accepted blob disposition plan under explicit authorization. + +Two effects, in one order that cannot be reversed: + +1. **Restore** every ``restore_required`` member into its ordinary spool + through the production receiver that admission already reads. Restoration + never touches the physical blob: the historical carrier survives this + module unconditionally, so a crash at any boundary leaves at least one + verified copy. +2. **Delete** ``source_present`` and ``superseded_prefix`` members through + the canonical blob-GC seam, which owns publisher exclusion, the final + locked liveness recheck, and crash-consistent generation intent. + +The plan is a capability, not a worklist. This module makes no classification +judgment: every member's proof is revalidated immediately before its effect, +and any drift — a changed source, a changed object, a new referent, a +different digest, a different denominator — invalidates the whole plan and +returns control to compilation. + +Deletion is bounded to unreferenced members by construction. A member whose +content is proven at its source but which a durable row still references +stays on disk; removing it is the reference owner's decision, and the GC seam +refuses it anyway. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path + +from polylogue.maintenance.blob_disposition import ( + BlobDisposition, + BlobDispositionContext, + BlobDispositionMember, + BlobDispositionPlan, + RestorationDestination, +) + +TOOL_VERSION = "blob-disposition-apply-v1" + + +class DispositionApplyError(RuntimeError): + """Raised when an apply cannot prove its exact authorized effect set.""" + + +class MemberOutcome(StrEnum): + """One terminal outcome per plan member. There is no unknown outcome.""" + + RESTORED = "restored" + RESTORATION_ALREADY_PRESENT = "restoration_already_present" + DELETED = "deleted" + RETAINED_REFERENCED = "retained_referenced" + RETAINED_ABSENT = "retained_absent" + BLOCKED = "blocked" + + +@dataclass(frozen=True, slots=True) +class MemberResult: + blob_hash: str + outcome: MemberOutcome + detail: str = "" + + def to_dict(self) -> dict[str, str]: + return {"blob_hash": self.blob_hash, "outcome": self.outcome.value, "detail": self.detail} + + +@dataclass(frozen=True, slots=True) +class DispositionApplyReceipt: + """Complete before/after evidence, derived only from member outcomes.""" + + tool_version: str + plan_digest: str + archive_root: str + blob_root: str + dry_run: bool + results: tuple[MemberResult, ...] + reclaimed_bytes: int = 0 + blockers: tuple[str, ...] = () + + @property + def counts(self) -> dict[str, int]: + counts = {outcome.value: 0 for outcome in MemberOutcome} + for result in self.results: + counts[result.outcome.value] += 1 + return counts + + @property + def ok(self) -> bool: + return not self.blockers and self.counts[MemberOutcome.BLOCKED.value] == 0 + + def to_dict(self) -> dict[str, object]: + return { + "tool_version": self.tool_version, + "plan_digest": self.plan_digest, + "archive_root": self.archive_root, + "blob_root": self.blob_root, + "dry_run": self.dry_run, + "ok": self.ok, + "counts": self.counts, + "reclaimed_bytes": self.reclaimed_bytes, + "blockers": list(self.blockers), + "results": [result.to_dict() for result in self.results], + } + + +def _revalidate(member: BlobDispositionMember, *, context: BlobDispositionContext) -> str | None: + """Re-derive the member's own proof at the moment of effect.""" + path = context.blob_store.blob_path(member.blob_hash) + if not path.is_file(): + return "physical object vanished between planning and apply" + try: + size_bytes = path.stat().st_size + except OSError as exc: + return f"physical object became unreadable: {exc}" + if size_bytes != member.size_bytes: + return f"physical object changed size {member.size_bytes} -> {size_bytes}" + referenced_now = member.blob_hash in context.referenced_hashes + if referenced_now and not member.referenced: + return "a new durable reference appeared after planning" + if member.disposition is BlobDisposition.RESTORE_REQUIRED: + for prover in context.provers: + if prover.prove(member.blob_hash, path, size_bytes) is not None: + return "a source proof appeared after planning; restoration is no longer justified" + return None + expected = member.proof + if expected is None: + return "member carries no proof to revalidate" + for prover in context.provers: + if prover.name != expected.prover: + continue + proof = prover.prove(member.blob_hash, path, size_bytes) + if proof is None: + return f"{expected.prover} no longer proves this object at its source" + if proof.source_path != expected.source_path or proof.mode is not expected.mode: + return f"{expected.prover} now proves a different source or mode" + return None + return f"prover {expected.prover} is not available at apply time" + + +def _resident_hook_event(spool_root: Path, event_id: str) -> Path | None: + """Locate an event anywhere in the spool, not only in today's shard. + + ``enqueue_hook_event`` shards by the current day and only refuses a + collision inside that shard, so a same-identity event spooled on another + day would be delivered twice. + """ + for candidate in sorted(spool_root.rglob(f"{event_id}.json")): + if candidate.is_file(): + return candidate + return None + + +def _restore_hook_event(member: BlobDispositionMember, *, path: Path, spool_root: Path) -> MemberResult: + from polylogue.sources.hooks import ( + HookSpoolRecordError, + enqueue_hook_event, + read_hook_spool_record, + ) + + try: + envelope = json.loads(path.read_bytes()) + except (OSError, json.JSONDecodeError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"carrier is not a readable envelope: {exc}") + if not isinstance(envelope, dict): + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "carrier envelope is not an object") + event_id = envelope.get("event_id") + if not isinstance(event_id, str) or not event_id: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "carrier envelope has no event identity") + resident = _resident_hook_event(spool_root, event_id) + if resident is not None: + try: + existing = read_hook_spool_record(resident) + except HookSpoolRecordError as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"destination is unreadable: {exc}") + if existing != envelope: + return MemberResult( + member.blob_hash, + MemberOutcome.BLOCKED, + "destination holds a different event under the same identity", + ) + return MemberResult(member.blob_hash, MemberOutcome.RESTORATION_ALREADY_PRESENT, str(resident)) + try: + published = enqueue_hook_event( + event_type=str(envelope["event_type"]), + session_id=str(envelope["session_id"]), + provider=str(envelope["provider"]), + timestamp=str(envelope["timestamp"]), + payload=dict(envelope["payload"]), + root=spool_root, + event_id=str(envelope["event_id"]), + ) + except (KeyError, TypeError, HookSpoolRecordError, OSError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"ordinary spool admission refused: {exc}") + try: + restored = read_hook_spool_record(published) + except HookSpoolRecordError as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"restored file does not read back: {exc}") + if restored != envelope: + return MemberResult( + member.blob_hash, + MemberOutcome.BLOCKED, + "destination holds a different event under the same identity", + ) + _fsync_directory(published.parent) + return MemberResult(member.blob_hash, MemberOutcome.RESTORED, str(published)) + + +def _restore_browser_capture(member: BlobDispositionMember, *, path: Path, spool_root: Path) -> MemberResult: + from pydantic import ValidationError + + from polylogue.browser_capture.models import BrowserCaptureEnvelope + from polylogue.browser_capture.receiver import ( + BrowserCaptureSpoolConflictError, + SpoolQuotaExceededError, + capture_artifact_path, + capture_dedup_content_hash, + write_capture_envelope_bytes, + ) + + try: + raw = path.read_bytes() + envelope = BrowserCaptureEnvelope.model_validate_json(raw) + except (OSError, ValidationError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"carrier is not a valid capture: {exc}") + destination = capture_artifact_path(envelope, spool_root) + if destination.is_file(): + try: + existing = BrowserCaptureEnvelope.model_validate_json(destination.read_bytes()) + except (OSError, ValidationError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"destination is unreadable: {exc}") + if capture_dedup_content_hash(existing) != capture_dedup_content_hash(envelope): + return MemberResult( + member.blob_hash, + MemberOutcome.BLOCKED, + "destination holds a different capture under the same identity", + ) + return MemberResult(member.blob_hash, MemberOutcome.RESTORATION_ALREADY_PRESENT, str(destination)) + try: + write_capture_envelope_bytes(raw, spool_path=spool_root) + except (BrowserCaptureSpoolConflictError, SpoolQuotaExceededError, OSError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"ordinary spool admission refused: {exc}") + if not destination.is_file(): + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "capture receiver published no artifact") + try: + restored = BrowserCaptureEnvelope.model_validate_json(destination.read_bytes()) + except (OSError, ValidationError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"restored file does not read back: {exc}") + if capture_dedup_content_hash(restored) != capture_dedup_content_hash(envelope): + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "restored capture is not content-equivalent") + _fsync_directory(destination.parent) + return MemberResult(member.blob_hash, MemberOutcome.RESTORED, str(destination)) + + +def _fsync_directory(path: Path) -> None: + """Persist the atomic rename's directory entry before claiming success.""" + try: + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + except OSError: + return + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def restore_plan_members( + plan: BlobDispositionPlan, + *, + context: BlobDispositionContext, + hook_spool_root: Path, + browser_capture_spool: Path, + dry_run: bool = True, +) -> tuple[MemberResult, ...]: + """Restore every sole-copy carrier into its ordinary spool. + + This never deletes or modifies the historical carrier, so an interruption + at any point leaves the blob intact and the operation resumable. + """ + results: list[MemberResult] = [] + for member in plan.members_for(BlobDisposition.RESTORE_REQUIRED): + drift = _revalidate(member, context=context) + if drift is not None: + results.append(MemberResult(member.blob_hash, MemberOutcome.BLOCKED, drift)) + continue + if member.restoration is None: + results.append( + MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "restore_required member names no destination") + ) + continue + if dry_run: + results.append( + MemberResult( + member.blob_hash, + MemberOutcome.RESTORED, + f"would restore to {member.restoration.destination.value}", + ) + ) + continue + path = context.blob_store.blob_path(member.blob_hash) + if member.restoration.destination is RestorationDestination.HOOK_EVENT_SPOOL: + results.append(_restore_hook_event(member, path=path, spool_root=hook_spool_root)) + else: + results.append(_restore_browser_capture(member, path=path, spool_root=browser_capture_spool)) + return tuple(results) + + +def _authorization_blockers( + plan: BlobDispositionPlan, + *, + authorized_digest: str, + context: BlobDispositionContext, +) -> tuple[str, ...]: + blockers: list[str] = [] + if not plan.accepted: + blockers.append(f"plan is not acceptable: {plan.unresolved_count} unresolved members") + actual = plan.digest() + if actual != authorized_digest: + blockers.append(f"authorized digest {authorized_digest[:16]} does not match plan digest {actual[:16]}") + if str(context.blob_store.root) != plan.blob_root: + blockers.append(f"plan blob namespace {plan.blob_root} is not the namespace being applied") + referenced_present = len(context.referenced_hashes & {member.blob_hash for member in plan.members}) + if referenced_present != plan.denominator.referenced_present_count: + blockers.append( + "referenced-and-present denominator drifted " + f"{plan.denominator.referenced_present_count} -> {referenced_present}" + ) + return tuple(blockers) + + +def apply_disposition_plan( + plan: BlobDispositionPlan, + *, + context: BlobDispositionContext, + authorized_digest: str, + source_db: Path, + index_db: Path, + hook_spool_root: Path, + browser_capture_spool: Path, + writer_block_reason: str | None = None, + dry_run: bool = True, +) -> DispositionApplyReceipt: + """Restore, then delete, exactly what the authorized plan names.""" + blockers = list(_authorization_blockers(plan, authorized_digest=authorized_digest, context=context)) + if writer_block_reason is not None and not dry_run: + blockers.append(f"an archive writer is active: {writer_block_reason}") + if blockers: + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=(), + blockers=tuple(blockers), + ) + + results: list[MemberResult] = list( + restore_plan_members( + plan, + context=context, + hook_spool_root=hook_spool_root, + browser_capture_spool=browser_capture_spool, + dry_run=dry_run, + ) + ) + if any(result.outcome is MemberOutcome.BLOCKED for result in results): + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=tuple(results), + blockers=("restoration did not complete; no deletion was attempted",), + ) + + removable: list[BlobDispositionMember] = [] + for disposition in (BlobDisposition.SOURCE_PRESENT, BlobDisposition.SUPERSEDED_PREFIX): + for member in plan.members_for(disposition): + drift = _revalidate(member, context=context) + if drift is not None: + results.append(MemberResult(member.blob_hash, MemberOutcome.BLOCKED, drift)) + continue + if member.referenced or member.blob_hash in context.referenced_hashes: + results.append( + MemberResult( + member.blob_hash, + MemberOutcome.RETAINED_REFERENCED, + "content is proven at its source but a durable row still references the object", + ) + ) + continue + removable.append(member) + + if any(result.outcome is MemberOutcome.BLOCKED for result in results): + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=tuple(results), + blockers=("member revalidation failed; no deletion was attempted",), + ) + + # An empty removable set has no effect to serialize: entering the GC seam + # would only report its own unmet preconditions as this plan's blockers. + if dry_run or not removable: + results.extend( + MemberResult(member.blob_hash, MemberOutcome.DELETED, "would unlink through the blob-GC seam") + for member in removable + ) + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=tuple(results), + reclaimed_bytes=sum(member.size_bytes for member in removable) if dry_run else 0, + ) + + from polylogue.storage.blob_gc import unlink_unreferenced_blob_hashes_under_exclusion + + deleted, reclaimed, errors = unlink_unreferenced_blob_hashes_under_exclusion( + source_db, + index_db, + context.blob_store.root, + {member.blob_hash for member in removable}, + ) + for member in removable: + if context.blob_store.blob_path(member.blob_hash).exists(): + results.append( + MemberResult(member.blob_hash, MemberOutcome.RETAINED_ABSENT, "the GC seam declined this member") + ) + else: + results.append(MemberResult(member.blob_hash, MemberOutcome.DELETED, "")) + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=False, + results=tuple(results), + reclaimed_bytes=reclaimed, + blockers=tuple(errors), + ) + + +def write_receipt(path: Path, receipt: DispositionApplyReceipt) -> None: + """Publish an append-only receipt, durably, before returning success.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.partial") + payload = json.dumps(receipt.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + "\n" + with temporary.open("w", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + _fsync_directory(path.parent) + + +__all__ = [ + "TOOL_VERSION", + "DispositionApplyError", + "DispositionApplyReceipt", + "MemberOutcome", + "MemberResult", + "apply_disposition_plan", + "restore_plan_members", + "write_receipt", +] diff --git a/polylogue/maintenance/embedding_preservation.py b/polylogue/maintenance/embedding_preservation.py index 7fe803af32..9e76736aff 100644 --- a/polylogue/maintenance/embedding_preservation.py +++ b/polylogue/maintenance/embedding_preservation.py @@ -4,10 +4,17 @@ import hashlib import json +import os import sqlite3 +import tempfile +from collections.abc import Iterator, Sequence +from contextlib import closing from dataclasses import asdict, dataclass +from enum import StrEnum from pathlib import Path +from polylogue.core.durable_fs import sync_directory, write_once +from polylogue.storage.sqlite.archive_tiers.embeddings import EMBEDDING_DIMENSION from polylogue.storage.sqlite.sqlite_vec_extension import try_load_sqlite_vec _VECTOR_TABLES = ( @@ -20,6 +27,25 @@ ) _CURRENT_HASH_COLUMN = "vector_derivation_hash" _LEGACY_HASH_COLUMN = "embedding_input_hash" +_META_FIELDS = ("model", "dimension", "embedded_at_ms", "recipe_hash", "output_contract_hash") +# Hashes per IN list. Bounded by the connection's own variable limit, which is +# 999 on a default SQLite build and must never be assumed larger. +_MAX_HASH_BATCH = 500 + + +class RestoreMissReason(StrEnum): + """Why a wanted hash did not restore.""" + + METADATA_ABSENT = "metadata_absent" + METADATA_INCOMPLETE = "metadata_incomplete" + VECTOR_ABSENT = "vector_absent" + + +@dataclass(frozen=True, slots=True) +class RestoreMiss: + input_hash: str + reason: RestoreMissReason + detail: str = "" @dataclass(frozen=True, slots=True) @@ -30,7 +56,7 @@ class EmbeddingPreservationReceipt: vector_rows: int table_set_digest: str restored_hashes: int = 0 - missing_hashes: tuple[str, ...] = () + misses: tuple[RestoreMiss, ...] = () def _connect(path: Path, *, readonly: bool) -> sqlite3.Connection: @@ -56,8 +82,12 @@ def _table_digest(conn: sqlite3.Connection) -> tuple[str, dict[str, int]]: return digest.hexdigest(), counts +def _columns(conn: sqlite3.Connection, table: str) -> set[str]: + return {str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})")} + + def _hash_column(conn: sqlite3.Connection, table: str) -> str: - columns = {str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})")} + columns = _columns(conn, table) if _CURRENT_HASH_COLUMN in columns: return _CURRENT_HASH_COLUMN if _LEGACY_HASH_COLUMN in columns: @@ -65,19 +95,67 @@ def _hash_column(conn: sqlite3.Connection, table: str) -> str: raise RuntimeError(f"{table} has no supported embedding hash column") +def _hash_batches(conn: sqlite3.Connection, values: Sequence[bytes]) -> Iterator[Sequence[bytes]]: + """Chunk host parameters below this connection's own variable limit.""" + size = max(1, min(_MAX_HASH_BATCH, conn.getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER))) + for start in range(0, len(values), size): + yield values[start : start + size] + + +def _receipt_path(copy_path: Path) -> Path: + return copy_path.with_suffix(copy_path.suffix + ".receipt.json") + + +def _fsync_file(path: Path) -> None: + handle = os.open(path, os.O_RDONLY) + try: + os.fsync(handle) + finally: + os.close(handle) + + +def _fsync_directory(path: Path) -> None: + sync_directory(path) + + def preserve_embedding_vectors(source: str | Path, destination: str | Path) -> EmbeddingPreservationReceipt: - """Checkpoint-copy an embeddings database and record its vector population.""" + """Checkpoint-copy an embeddings database and record its vector population. + + The copy is built in a private temporary file and renamed into place only + once the backup has finished and the receipt has been derived from the + finished copy, so a file at the destination path is always a whole copy + that its receipt describes. + """ source_path = Path(source).absolute() destination_path = Path(destination).absolute() if source_path == destination_path: raise ValueError("embedding preservation source and copy must differ") destination_path.parent.mkdir(parents=True, exist_ok=True) - with _connect(source_path, readonly=True) as source_conn: - digest, counts = _table_digest(source_conn) - if destination_path.exists(): - raise FileExistsError(destination_path) - with sqlite3.connect(destination_path) as copy_conn: + if destination_path.exists(): + raise FileExistsError(destination_path) + handle, partial_name = tempfile.mkstemp( + dir=destination_path.parent, prefix=f".{destination_path.name}.", suffix=".partial" + ) + os.close(handle) + partial = Path(partial_name) + try: + with ( + closing(_connect(source_path, readonly=True)) as source_conn, + closing(sqlite3.connect(partial)) as copy_conn, + ): source_conn.backup(copy_conn) + # The copy inherits the source's journal mode, and a rename moves + # only the main file: the archived copy is made self-contained so + # it can never be separated from a WAL holding its content. + copy_conn.execute("PRAGMA journal_mode=DELETE").fetchall() + with closing(_connect(partial, readonly=True)) as copy_reader: + digest, counts = _table_digest(copy_reader) + _fsync_file(partial) + os.replace(partial, destination_path) + _fsync_directory(destination_path.parent) + except BaseException: + partial.unlink(missing_ok=True) + raise receipt = EmbeddingPreservationReceipt( source=str(source_path), copy=str(destination_path), @@ -85,12 +163,77 @@ def preserve_embedding_vectors(source: str | Path, destination: str | Path) -> E vector_rows=counts["message_embeddings"], table_set_digest=digest, ) - destination_path.with_suffix(destination_path.suffix + ".receipt.json").write_text( - json.dumps(asdict(receipt), indent=2, sort_keys=True) + "\n", encoding="utf-8" + write_once( + _receipt_path(destination_path), + (json.dumps(asdict(receipt), indent=2, sort_keys=True) + "\n").encode("utf-8"), ) return receipt +@dataclass(frozen=True, slots=True) +class _PreservedMetadata: + """A preserved row in the shape the current tier requires.""" + + model: str + dimension: int + embedded_at_ms: int | None + recipe_hash: bytes + output_contract_hash: bytes + + +def _validated_metadata(fields: dict[str, object]) -> _PreservedMetadata | str: + """The row as the current tier requires it, or the field that disqualifies it. + + ``message_embeddings_meta`` is complete by schema: a preserved row whose + model, dimension, or derivation identity is absent describes an output + nobody can vouch for, so it cannot stand in for a fresh embedding. + """ + model = fields.get("model") + if not isinstance(model, str) or not model: + return "model" + dimension = fields.get("dimension") + if not isinstance(dimension, int) or dimension != EMBEDDING_DIMENSION: + return "dimension" + identities: dict[str, bytes] = {} + for name in ("recipe_hash", "output_contract_hash"): + value = fields.get(name) + if not isinstance(value, (bytes, bytearray, memoryview)) or len(value) != 32: + return name + identities[name] = bytes(value) + embedded_at_ms = fields.get("embedded_at_ms") + return _PreservedMetadata( + model=model, + dimension=dimension, + embedded_at_ms=embedded_at_ms if isinstance(embedded_at_ms, int) else None, + recipe_hash=identities["recipe_hash"], + output_contract_hash=identities["output_contract_hash"], + ) + + +def _preserved_metadata( + conn: sqlite3.Connection, hash_column: str, projection: Sequence[str], batch: Sequence[bytes] +) -> dict[bytes, dict[str, object]]: + columns = ", ".join((hash_column, *projection)) + placeholders = ",".join("?" for _ in batch) + rows = conn.execute( + f"SELECT {columns} FROM message_embeddings_meta WHERE {hash_column} IN ({placeholders})", + tuple(batch), + ).fetchall() + return {bytes(row[0]): dict(zip(projection, row[1:], strict=True)) for row in rows} + + +def _preserved_vectors( + conn: sqlite3.Connection, hash_column: str, batch: Sequence[bytes] +) -> dict[bytes, tuple[object, object]]: + addresses = [value.hex() for value in batch] + placeholders = ",".join("?" for _ in addresses) + rows = conn.execute( + f"SELECT {hash_column}, embedding, model FROM message_embeddings WHERE {hash_column} IN ({placeholders})", + addresses, + ).fetchall() + return {bytes.fromhex(str(row[0])): (row[1], row[2]) for row in rows} + + def restore_embedding_vectors( destination: str | Path, preserved_copy: str | Path, @@ -98,64 +241,84 @@ def restore_embedding_vectors( ) -> EmbeddingPreservationReceipt: """Import preserved vectors for ``input_hashes`` into a fresh embeddings DB. - Metadata and vectors are write-once by input hash. Refs and lifecycle rows - remain owned by the fresh database and are created by normal convergence. + Metadata and vectors are write-once by input hash and are written together + in one transaction: a metadata row is the tier's reuse signal, so it may + never exist without the vector at its address. A hash counts as restored + only once both rows are present; every other outcome is an enumerated miss + carrying its cause. Refs and lifecycle rows remain owned by the fresh + database and are created by normal convergence. """ destination_path = Path(destination).absolute() copy_path = Path(preserved_copy).absolute() wanted = sorted(input_hashes) - with _connect(destination_path, readonly=False) as target, _connect(copy_path, readonly=True) as source: - source_meta_hash = _hash_column(source, "message_embeddings_meta") - source_vector_hash = _hash_column(source, "message_embeddings") - if wanted: - placeholders = ",".join("?" for _ in wanted) - rows = source.execute( - f"SELECT {source_meta_hash}, model, dimension, embedded_at_ms, recipe_hash, output_contract_hash " - f"FROM message_embeddings_meta WHERE {source_meta_hash} IN ({placeholders})", - wanted, - ).fetchall() - found = {bytes(row[0]) for row in rows} - for row in rows: + restored = 0 + misses: list[RestoreMiss] = [] + with ( + closing(_connect(destination_path, readonly=False)) as target, + closing(_connect(copy_path, readonly=True)) as source, + ): + meta_hash_column = _hash_column(source, "message_embeddings_meta") + vector_hash_column = _hash_column(source, "message_embeddings") + projection = [name for name in _META_FIELDS if name in _columns(source, "message_embeddings_meta")] + for batch in _hash_batches(source, wanted): + preserved = _preserved_metadata(source, meta_hash_column, projection, batch) + vectors = _preserved_vectors(source, vector_hash_column, batch) + for value in batch: + fields = preserved.get(value) + if fields is None: + misses.append(RestoreMiss(value.hex(), RestoreMissReason.METADATA_ABSENT)) + continue + record = _validated_metadata(fields) + if isinstance(record, str): + misses.append(RestoreMiss(value.hex(), RestoreMissReason.METADATA_INCOMPLETE, record)) + continue + vector = vectors.get(value) + if vector is None: + misses.append(RestoreMiss(value.hex(), RestoreMissReason.VECTOR_ABSENT)) + continue + target.execute( + "INSERT OR IGNORE INTO message_embeddings (vector_derivation_hash, embedding, model) " + "VALUES (?, ?, ?)", + (value.hex(), vector[0], vector[1]), + ) target.execute( "INSERT OR IGNORE INTO message_embeddings_meta " "(vector_derivation_hash, model, dimension, embedded_at_ms, recipe_hash, output_contract_hash) " "VALUES (?, ?, ?, ?, ?, ?)", - row, - ) - vector = source.execute( - f"SELECT embedding, model FROM message_embeddings WHERE {source_vector_hash} = ?", - (bytes(row[0]).hex(),), - ).fetchone() - if vector is None: - continue - target.execute( - "INSERT OR IGNORE INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", - (bytes(row[0]).hex(), vector[0], vector[1]), + ( + value, + record.model, + record.dimension, + record.embedded_at_ms, + record.recipe_hash, + record.output_contract_hash, + ), ) + restored += 1 target.commit() - else: - found = set() digest, counts = _table_digest(source) - missing = tuple(value.hex() for value in wanted if value not in found) return EmbeddingPreservationReceipt( source=str(copy_path), copy=str(destination_path), metadata_rows=counts["message_embeddings_meta"], vector_rows=counts["message_embeddings"], table_set_digest=digest, - restored_hashes=len(found), - missing_hashes=missing, + restored_hashes=restored, + misses=tuple(misses), ) def delete_preserved_copy(path: str | Path, *, receipt_path: str | Path | None = None) -> None: - """Delete a preservation copy only when an AC2 receipt authorizes it.""" + """Delete a preservation copy only when an AC2 receipt proves it is this copy. + + The receipt must name this file and carry the table-set digest the copy + still has, so a receipt filed for one copy can never authorize deleting + another, nor a copy that has changed since it was proven. + """ copy_path = Path(path).absolute() if not copy_path.is_file() or copy_path.is_symlink(): raise FileNotFoundError(copy_path) - if receipt_path is None: - receipt_path = copy_path.with_suffix(copy_path.suffix + ".receipt.json") - receipt = Path(receipt_path).absolute() + receipt = Path(receipt_path).absolute() if receipt_path is not None else _receipt_path(copy_path) if not receipt.is_file() or receipt.is_symlink(): raise FileNotFoundError(receipt) try: @@ -164,12 +327,21 @@ def delete_preserved_copy(path: str | Path, *, receipt_path: str | Path | None = raise ValueError("preservation deletion receipt is not valid JSON") from exc if proof.get("ac2_passed") is not True: raise ValueError("preservation copy requires an AC2-passed receipt before deletion") + named = proof.get("copy") + if not isinstance(named, str) or Path(named).absolute() != copy_path: + raise ValueError("preservation receipt names a different copy") + with closing(_connect(copy_path, readonly=True)) as conn: + digest, _counts = _table_digest(conn) + if digest != proof.get("table_set_digest"): + raise ValueError("preservation copy no longer matches its receipt digest") copy_path.unlink() receipt.unlink() __all__ = [ "EmbeddingPreservationReceipt", + "RestoreMiss", + "RestoreMissReason", "delete_preserved_copy", "preserve_embedding_vectors", "restore_embedding_vectors", diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 75028ee520..b2688e0d90 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -402,28 +402,6 @@ def _message_payload(message: ParsedMessage, fields: frozenset[str]) -> dict[str return payload -def _message_reference_payload(message: ParsedMessage) -> dict[str, JSONValue]: - """Extend the content payload with the reference identity of media blocks. - - An ``image``/``document`` block's metadata names what the turn cites (a - Drive file id, an asset pointer, an inline-content digest). Two id-less, - timestamp-less, text-less turns that cite different files are different - turns, but ``_content_block_payload`` deliberately keeps metadata out of - the session content hash, so this payload is a private owner - discriminator only: it decides ownership after the content payload has - already collided and never feeds a stored hash. - """ - payload = _message_comparison_payload(message) - references: list[JSONValue] = [ - hash_payload(_normalize_nested_for_hash(dict(block.metadata))) - for block in message.blocks - if block.type in (BlockType.IMAGE, BlockType.DOCUMENT) and block.metadata - ] - if references: - payload["block_references"] = references - return payload - - def _message_semantic_payload(message: ParsedMessage) -> dict[str, JSONValue]: """Build the complete semantic payload used by session content hashing.""" return _message_payload(message, _HASHED_FIELDS["ParsedMessage"]) @@ -519,6 +497,13 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol reorder-stable evidence when their content is identical. If neither distinguishes the occurrences, the duplicate remains typed ambiguity instead of receiving a position-derived identity. + + The content discriminator covers a media block's ``metadata`` -- what an + ``image``/``document`` turn cites (a Drive file id, an asset pointer, an + inline-content digest). AI Studio exports carry runs of id-less, + text-less turns that share one timestamp and differ only there, so + ``metadata`` is the sole evidence keeping their attachments ownable + (polylogue-prjai). """ revision_ids = tuple(_message_revision_match_id(message) for message in messages) revision_counts = Counter(revision_ids) @@ -526,25 +511,17 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_comparison_payload(message))}" for message in messages ) content_counts = Counter(content_ids) - reference_ids = tuple( - f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_reference_payload(message))}" for message in messages - ) - reference_counts = Counter(reference_ids) coordinates = tuple(_message_owner_coordinate(message, index) for index, message in enumerate(messages)) stable_counts = Counter(coordinate.stable_key for coordinate in coordinates if coordinate.stable_key is not None) keys: list[str] = [] - for revision_id, content_id, reference_id, coordinate in zip( - revision_ids, content_ids, reference_ids, coordinates, strict=True - ): + for revision_id, content_id, coordinate in zip(revision_ids, content_ids, coordinates, strict=True): if coordinate.stable_key is not None and stable_counts[coordinate.stable_key] == 1: key = coordinate.stable_key elif revision_counts[revision_id] == 1: key = revision_id elif content_counts[content_id] == 1: key = content_id - elif reference_counts[reference_id] == 1: - key = reference_id elif coordinate.stable_key is not None: key = coordinate.stable_key else: diff --git a/polylogue/pipeline/services/archive_ingest.py b/polylogue/pipeline/services/archive_ingest.py index 10b75d84de..4ba3876fa0 100644 --- a/polylogue/pipeline/services/archive_ingest.py +++ b/polylogue/pipeline/services/archive_ingest.py @@ -3,9 +3,14 @@ from __future__ import annotations import json +import time import zipfile +from collections.abc import Callable, Mapping from concurrent.futures import as_completed +from contextlib import suppress +from dataclasses import dataclass from datetime import UTC, datetime +from functools import partial from pathlib import Path from typing import Any @@ -17,8 +22,10 @@ from polylogue.logging import get_logger from polylogue.pipeline.services.parsing_models import ParseResult from polylogue.pipeline.services.process_pool import ( + PoolKind, process_pool_executor, resolve_archive_ingest_dispatch, + resolve_parse_worker_count, ) from polylogue.sources.decoder_zip import ( ZipBombError, @@ -26,6 +33,7 @@ open_bounded_zip_entry, zip_entry_session_artifact, ) +from polylogue.sources.dispatch import require_positive_conversational_evidence from polylogue.sources.parsers import antigravity from polylogue.sources.parsers.base import ParsedSession, RawSessionData from polylogue.sources.source_parsing import ( @@ -57,6 +65,41 @@ POST_COMMIT_UPKEEP_REASON = "archive_ingest_commit" +@dataclass(frozen=True, slots=True) +class _ParseSubmission: + """One walked source file and everything its parse worker needs.""" + + source: Source + path: Path + file_mtime: Any + sidecar_data: Mapping[str, Any] | None + + +def _record_stage(result: ParseResult, name: str, started_at: float) -> None: + """Accumulate one ingest phase into the run's stage ledger. + + Keeps the walk and parse phases inside the same ``append.*`` namespace the + archive write path already reports, so the ledger sums toward wall time + instead of describing the write alone. + """ + key = f"append.{name}" + result.stage_timings_s[key] = result.stage_timings_s.get(key, 0.0) + (time.perf_counter() - started_at) + + +def _submission_payload_bytes(submissions: list[_ParseSubmission]) -> int: + """Total on-disk size of the files this walk will parse. + + Sizes the parse dispatch. A path that vanished between the walk and here + contributes nothing, which biases the plan toward sequential -- the safe + direction, since the pool only pays off above the byte tiers. + """ + total = 0 + for submission in submissions: + with suppress(OSError): + total += submission.path.stat().st_size + return total + + def _commit_batch_message_threshold() -> int: from polylogue.config import load_polylogue_config @@ -113,6 +156,10 @@ async def parse_sources_archive( resolve out-of-order. Blob writes from workers are content-addressed and atomic, so concurrent worker writes are process-safe. + The pool is sized from the work the walk actually found + (:func:`resolve_archive_ingest_dispatch`), so a small walk parses + in-process instead of paying a spawn per worker for it. + ``parse_workers`` overrides the ambient/env-resolved worker count for this call only (used by the demo seeder to force sequential parsing -- see ``polylogue/demo/seed.py``). ``None`` preserves the normal @@ -122,7 +169,7 @@ async def parse_sources_archive( acquired_at_ms = int(datetime.now(UTC).timestamp() * 1000) threshold = _commit_batch_message_threshold() batched = threshold > 0 - workers = resolve_archive_ingest_dispatch(parse_workers=parse_workers).worker_count + workers = resolve_parse_worker_count() if parse_workers is None else max(1, parse_workers) blob_root = archive_root / "blob" from polylogue.storage.blob_publication import ArchiveBlobPublisher @@ -158,6 +205,20 @@ async def write_pair( raw_data: RawSessionData | None, session: ParsedSession, ) -> None: + # polylogue-b508: a session requires positive evidence of a + # conversation. The daemon decode worker, live batch convergence, + # the incremental append route, and offline replay each apply this + # law right after dispatch returns; this one-shot importer is the + # remaining production write path and must agree, or a document + # that merely satisfies dispatch's loose messages-list shape is + # written as a session keyed on its own filename stem -- identity + # the discovery walk invented, not identity a provider asserted. + if not require_positive_conversational_evidence( + [session], + provider=session.source_name, + source_path=_archive_raw_source_path(raw_data, source), + ): + return session = normalize_session_timestamps( session, fallback_timestamp=raw_data.file_mtime if raw_data is not None else None, @@ -322,53 +383,91 @@ async def write_pair( await write_pair(source, raw_data, session) failed = 0 - total_paths = 0 - with process_pool_executor(max_workers=workers) as pool: - future_to_source: dict[Any, tuple[Source, Path]] = {} - for source in sources: - walk = _setup_source_walk( - source, - cursor_state=None, - include_mtime=True, - known_mtimes=None, - discover_sidecars=True, - blob_store=parse_blob_publisher, - ) - if walk is None: + submissions: list[_ParseSubmission] = [] + walk_started_at = time.perf_counter() + for source in sources: + walk = _setup_source_walk( + source, + cursor_state=None, + include_mtime=True, + known_mtimes=None, + discover_sidecars=True, + blob_store=parse_blob_publisher, + ) + if walk is None: + continue + for path, file_mtime in walk.paths_to_process: + if ( + Provider.from_string(source.name) is Provider.ANTIGRAVITY + and antigravity.classify_source_path(path).role + is antigravity.AntigravitySourceRole.CONVERSATION_PROTOBUF + ): continue - for path, file_mtime in walk.paths_to_process: - if ( - Provider.from_string(source.name) is Provider.ANTIGRAVITY - and antigravity.classify_source_path(path).role - is antigravity.AntigravitySourceRole.CONVERSATION_PROTOBUF - ): - continue - future = pool.submit( - _parse_source_path_worker, - str(path), - file_mtime, - source.name, - walk.sidecar_data, - True, - str(blob_root), - str(archive_root / "source.db"), + submissions.append(_ParseSubmission(source, path, file_mtime, walk.sidecar_data)) + total_paths = len(submissions) + _record_stage(result, "walk", walk_started_at) + + def _parse_args(submission: _ParseSubmission) -> tuple[Any, ...]: + return ( + str(submission.path), + submission.file_mtime, + submission.source.name, + submission.sidecar_data, + True, + str(blob_root), + str(archive_root / "source.db"), + ) + + async def consume(source: Source, path: Path, produce: Callable[[], Any]) -> None: + nonlocal failed + parse_started_at = time.perf_counter() + try: + pairs = produce() + except Exception as exc: + # Worker error isolation: one bad file must not kill the + # run. Mirror the sequential iterator's failure handling. + _record_stage(result, "parse", parse_started_at) + failed += 1 + result.parse_failures += 1 + logger.error("Failed to parse %s in worker: %s", path, exc) + return + _record_stage(result, "parse", parse_started_at) + for raw_data, session in pairs: + await write_pair(source, raw_data, session) + + plan = resolve_archive_ingest_dispatch( + path_count=total_paths, + total_bytes=_submission_payload_bytes(submissions), + worker_ceiling=workers, + ) + if plan.pool_kind is PoolKind.SEQUENTIAL: + for submission in submissions: + await consume( + submission.source, + submission.path, + partial(_parse_source_path_worker, *_parse_args(submission)), + ) + elif submissions: + pool_started_at = time.perf_counter() + with process_pool_executor(max_workers=plan.worker_count) as pool: + future_to_source: dict[Any, tuple[Source, Path]] = { + pool.submit(_parse_source_path_worker, *_parse_args(submission)): ( + submission.source, + submission.path, ) - future_to_source[future] = (source, path) - total_paths += 1 - - for future in as_completed(future_to_source): - source, path = future_to_source[future] - try: - pairs = future.result() - except Exception as exc: - # Worker error isolation: one bad file must not kill the - # run. Mirror the sequential iterator's failure handling. - failed += 1 - result.parse_failures += 1 - logger.error("Failed to parse %s in worker: %s", path, exc) - continue - for raw_data, session in pairs: - await write_pair(source, raw_data, session) + for submission in submissions + } + _record_stage(result, "parse_pool", pool_started_at) + # Workers spawn lazily behind `submit`, so the parse itself + # is the wait between completions, not `future.result()`. + wait_started_at = time.perf_counter() + for future in as_completed(future_to_source): + _record_stage(result, "parse", wait_started_at) + source, path = future_to_source[future] + await consume(source, path, future.result) + wait_started_at = time.perf_counter() + shutdown_started_at = time.perf_counter() + _record_stage(result, "parse_pool", shutdown_started_at) if failed > 0: logger.warning( diff --git a/polylogue/pipeline/services/process_pool.py b/polylogue/pipeline/services/process_pool.py index 82af066770..7795816ed1 100644 --- a/polylogue/pipeline/services/process_pool.py +++ b/polylogue/pipeline/services/process_pool.py @@ -147,19 +147,28 @@ class ParseDispatchPlan: worker_count: int -def resolve_archive_ingest_dispatch(*, parse_workers: int | None = None) -> ParseDispatchPlan: - """Worker-count decision for ``archive_ingest.py``'s re-ingest file-walk parse. - - Unchanged formula: an explicit ``parse_workers`` override (clamped to at - least 1) wins; otherwise :func:`resolve_parse_worker_count` (CPU count, - ceiling adjusted for a free-threaded build). Always a process pool -- the - caller's own ``workers <= 1`` branch is the escape hatch to sequential, - preserved unchanged at the call site rather than folded into - :data:`PoolKind` here, since that branch also skips constructing the pool - context entirely (a real, not merely nominal, sequential path). +def resolve_archive_ingest_dispatch(*, path_count: int, total_bytes: int, worker_ceiling: int) -> ParseDispatchPlan: + """Pool-kind + worker-count decision for ``archive_ingest.py``'s file-walk parse. + + Sized by the work the walk actually found, on the same byte tiers as + :func:`resolve_ingest_batch_dispatch`: ``<= 8 MiB`` sequential, ``<= 64 + MiB`` capped at 4 workers, above that ``min(path_count, cpus, ceiling)``. + A spawn pool costs a fresh interpreter and a full ``polylogue`` import per + worker; below the first tier that setup exceeds the parse it replaces, and + a spawn failure under host pressure is absorbed by the driver's per-file + ``except`` as a silently dropped file rather than surfacing as an error. + + ``worker_ceiling`` is the caller's already-resolved + :func:`resolve_parse_worker_count` value, so the operator knob keeps one + home. A ceiling of 1 never reaches here: it selects the caller's + source-iterator escape hatch, which is a different route from the walk. """ - worker_count = resolve_parse_worker_count() if parse_workers is None else max(1, parse_workers) - return ParseDispatchPlan(PoolKind.PROCESS, worker_count) + if path_count <= 1 or total_bytes <= 8 * 1024 * 1024: + return ParseDispatchPlan(PoolKind.SEQUENTIAL, 1) + cpus = available_cpus() or 4 + if total_bytes <= 64 * 1024 * 1024: + return ParseDispatchPlan(PoolKind.PROCESS, max(1, min(path_count, cpus, worker_ceiling, 4))) + return ParseDispatchPlan(PoolKind.PROCESS, max(1, min(path_count, cpus, worker_ceiling))) def resolve_validation_dispatch(*, record_count: int) -> ParseDispatchPlan: diff --git a/polylogue/sources/dispatch.py b/polylogue/sources/dispatch.py index c560d092a3..3fce07ff1c 100644 --- a/polylogue/sources/dispatch.py +++ b/polylogue/sources/dispatch.py @@ -1693,11 +1693,13 @@ def require_positive_conversational_evidence( which already treats an empty session list as a recorded, bounded ``mark_raw_parse_failed`` outcome -- this filter reuses that existing "refused loudly" mechanism rather than inventing a new one), - ``sources/live/append_ingest.py`` (incremental append), and + ``sources/live/append_ingest.py`` (incremental append), ``sources/revision_backfill.py`` (offline replay/rebuild, alongside its own OriginSpec/``classify_artifact`` path-and-shape gate from polylogue-6mpy -- this filter catches the sibling case where the shape - is recognized but the parsed *content* still carries no message). + is recognized but the parsed *content* still carries no message), and + ``pipeline/services/archive_ingest.py`` (the one-shot importer behind + ``Polylogue.parse_file``/``parse_sources`` and the demo seeder). Measured against the live archive (2026-07-31, read-only query against ``index.db``/``source.db``): every verified zero-message diff --git a/polylogue/sources/hooks.py b/polylogue/sources/hooks.py index e11b1c5ef9..b17eaa4ce9 100644 --- a/polylogue/sources/hooks.py +++ b/polylogue/sources/hooks.py @@ -446,6 +446,18 @@ def _prune_empty_shards(shards: set[Path], pending_root: Path) -> None: shard.rmdir() +def read_hook_spool_record(path: Path) -> dict[str, object]: + """Read one spool file through the same validation acquisition applies. + + The record acquisition stores is not the file's bytes: ``observed_at_ms`` + is derived here, and serialization is independent on both sides. Any + comparison against stored hook material must go through this route rather + than compare bytes. + """ + + return _read_record(path) + + def _read_record(path: Path) -> dict[str, object]: try: value = json.loads(path.read_text(encoding="utf-8")) @@ -615,5 +627,6 @@ def _fsync_directory(path: Path) -> None: "enqueue_hook_event", "hook_spool_root", "pending_hook_spool_dir", + "read_hook_spool_record", "validate_hook_spool_topology", ] diff --git a/polylogue/storage/attachment_relink.py b/polylogue/storage/attachment_relink.py index 6d0791e7d1..d1a893a4bf 100644 --- a/polylogue/storage/attachment_relink.py +++ b/polylogue/storage/attachment_relink.py @@ -39,6 +39,7 @@ from enum import StrEnum from pathlib import Path +from polylogue.core.message_owner import MessageOwnerAmbiguityError from polylogue.logging import get_logger from polylogue.pipeline.ids import attachment_message_owner_key, message_owner_resolution from polylogue.pipeline.services.ingest_worker import IngestRecordResult, SessionWritePayload, ingest_record @@ -72,6 +73,10 @@ "matched raw content but the owning message is no longer present in the current index " "(session likely re-ingested without it since)" ) +_OWNER_AMBIGUOUS_REASON = ( + "the raw session reproduces this attachment but more than one message claims its owner " + "coordinate, so no ref may be guessed" +) @dataclass(frozen=True, slots=True) @@ -94,6 +99,7 @@ class RelinkableAttachment: class UnrecoverableAttachmentReason(StrEnum): NO_AUTHORITATIVE_RAW = "no_authoritative_raw" MESSAGE_MISSING = "message_missing" + OWNER_AMBIGUOUS = "owner_ambiguous" @dataclass(frozen=True, slots=True) @@ -379,7 +385,20 @@ def _match_session_payload( attachments_by_message: dict[str, list[ParsedAttachment]] = {} for attachment in payload.parsed_session.attachments: attachment_id = _attachment_id(session_id, attachment) - owner_key = attachment_message_owner_key(attachment, owner_resolution) + try: + owner_key = attachment_message_owner_key(attachment, owner_resolution) + except MessageOwnerAmbiguityError: + # The writer retains such an attachment as typed unowned evidence + # (write.py:_write_attachments), so its ref-less row reaches this + # scan. The ambiguity is the same at re-parse time: report it as a + # typed unrecoverable outcome rather than propagating out of the + # plan. + if attachment_id in pending: + ineligible_reasons.setdefault( + attachment_id, + (UnrecoverableAttachmentReason.OWNER_AMBIGUOUS, _OWNER_AMBIGUOUS_REASON), + ) + continue message_id = by_owner_key.get(owner_key) if owner_key is not None else None if message_id is None: if attachment_id in pending: diff --git a/tests/unit/devtools/test_query_execution_envelope.py b/tests/unit/devtools/test_query_execution_envelope.py new file mode 100644 index 0000000000..75a1f3160c --- /dev/null +++ b/tests/unit/devtools/test_query_execution_envelope.py @@ -0,0 +1,140 @@ +"""Tests for the live query execution envelope lab command.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import devtools.query_execution_envelope as envelope_module +from devtools.query_execution_envelope import ( + ResourceProbeUnavailableError, + _parse_proc_memory, + _proc_memory, + _temp_used_bytes, + measure_query_envelope, +) + + +def test_proc_memory_is_nonnegative() -> None: + rss, pss, swap = _proc_memory() + assert rss >= 0 + assert pss >= 0 + assert swap >= 0 + + +def test_temp_usage_missing_path_is_zero(tmp_path: Path) -> None: + assert _temp_used_bytes(tmp_path / "missing") == 0 + + +def test_parse_proc_memory_reads_every_declared_field() -> None: + rss, pss, swap = _parse_proc_memory("VmRSS:\t2048 kB\nVmSwap:\t4 kB\nPss:\t1024 kB\n") + + assert (rss, pss, swap) == (2048 * 1024, 1024 * 1024, 4 * 1024) + + +def test_parse_proc_memory_refuses_a_missing_field() -> None: + """A field procfs does not report is refused, never sampled as zero. + + Restoring a zero default makes this green and every declared limit + satisfiable without measuring anything. + """ + with pytest.raises(ResourceProbeUnavailableError, match="Pss"): + _parse_proc_memory("VmRSS:\t2048 kB\nVmSwap:\t4 kB\n") + + +async def test_measure_query_envelope_runs_the_declared_repetition_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The receipt covers every query round and all four resource dimensions.""" + + (tmp_path / "index.db").write_bytes(b"synthetic index") + calls = 0 + + class FakeEnvelope: + def model_dump(self, *, mode: str) -> dict[str, object]: + assert mode == "json" + return {"items": [{"group_key": "shell", "count": 1}]} + + class FakePolylogue: + def __init__(self, **_kwargs: object) -> None: + pass + + async def __aenter__(self) -> FakePolylogue: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def query_units(self, expression: str, *, limit: int) -> FakeEnvelope: + nonlocal calls + assert expression == "actions where tool:shell | group by tool | count" + assert limit == 100 + calls += 1 + return FakeEnvelope() + + monkeypatch.setattr(envelope_module, "Polylogue", FakePolylogue) + monkeypatch.setattr(envelope_module, "_proc_memory", lambda: (100, 80, 0)) + monkeypatch.setattr(envelope_module, "_temp_used_bytes", lambda _root: 100) + + receipt = await measure_query_envelope( + tmp_path, + warmup=0, + baseline_rounds=2, + sample_interval_s=0.001, + max_rss_bytes=100, + max_pss_bytes=80, + max_swap_growth_bytes=0, + max_temp_growth_bytes=0, + ) + + assert calls == 22 + assert receipt["status"] == "succeeded" + assert len(receipt["samples"]) == 22 + assert len(receipt["final_samples"]) == 3 + assert receipt["return_checks"] == {"rss": True, "pss": True, "swap": True, "temp": True} + assert receipt["absolute_checks"] == {"rss": True, "pss": True, "swap": True, "temp": True} + + +async def test_measure_query_envelope_fails_when_declared_rss_is_exceeded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The absolute RSS declaration is a real failure condition.""" + + (tmp_path / "index.db").write_bytes(b"synthetic index") + + class FakeEnvelope: + def model_dump(self, *, mode: str) -> dict[str, object]: + return {"items": []} + + class FakePolylogue: + def __init__(self, **_kwargs: object) -> None: + pass + + async def __aenter__(self) -> FakePolylogue: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def query_units(self, _expression: str, *, limit: int) -> FakeEnvelope: + assert limit == 100 + return FakeEnvelope() + + monkeypatch.setattr(envelope_module, "Polylogue", FakePolylogue) + monkeypatch.setattr(envelope_module, "_proc_memory", lambda: (100, 80, 0)) + monkeypatch.setattr(envelope_module, "_temp_used_bytes", lambda _root: 100) + + receipt = await measure_query_envelope( + tmp_path, + warmup=0, + baseline_rounds=1, + sample_interval_s=0.001, + max_rss_bytes=99, + max_pss_bytes=80, + max_swap_growth_bytes=0, + max_temp_growth_bytes=0, + ) + + assert receipt["status"] == "failed" + assert receipt["absolute_checks"]["rss"] is False diff --git a/tests/unit/maintenance/test_archive_verification.py b/tests/unit/maintenance/test_archive_verification.py index fb8f1767a5..3e1eed9352 100644 --- a/tests/unit/maintenance/test_archive_verification.py +++ b/tests/unit/maintenance/test_archive_verification.py @@ -19,7 +19,7 @@ HOOK_AUTHORITATIVE_LINK_METHOD, HOOK_CONTRADICTED_LINK_METHOD, ) -from polylogue.core.enums import ArtifactSupportStatus, Origin +from polylogue.core.enums import ArtifactSupportStatus, Origin, Provider, Role from polylogue.core.outcomes import OutcomeStatus from polylogue.maintenance.archive_verification import ( ArchiveVerificationCheck, @@ -31,10 +31,12 @@ verify_archive, ) from polylogue.sources.origin_specs import lowering_fingerprint, parser_fingerprint_for_origin +from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession from polylogue.storage.blob_store import BlobStore from polylogue.storage.sqlite.archive_tiers.bootstrap import ARCHIVE_TIER_SPECS, initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.archive_tiers.write import write_parsed_session_to_archive from polylogue.storage.sqlite.maintenance import analyze_planner_stats_tables from tests.infra.pathology_zoo import ( CLAUDE_VINTAGE_LIVE_PROOF_LOGICAL_SOURCE_KEY, @@ -1430,6 +1432,57 @@ def test_blob_reference_closure_rejects_acquired_attachment_without_ref(tmp_path assert check.evidence["acquired_attachment_missing_ref_count"] == 1 +def test_unowned_attachment_evidence_keeps_closure_and_coverage_clean(tmp_path: Path) -> None: + """The writer's typed-unowned attachment row is evidence, not archive debt. + + ``_write_attachments`` retains an attachment whose owner coordinate is + claimed by more than one message: the row is written with no + ``attachment_refs`` edge and no acquired bytes. Both required checks key + on acquired-and-unreferenced, so this shape must stay clean. + + Anti-vacuity: give the row ``acquisition_status = 'acquired'`` and both + checks turn ERROR (``test_blob_reference_closure_rejects_acquired_attachment_without_ref`` + and ``test_acquired_unreachable_attachment_debt_is_blocking`` pin that). + """ + _seed_coherent_archive(tmp_path) + conn = _connect(tmp_path / "index.db") + try: + session = ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="ambiguous-attachment-owner", + messages=[ParsedMessage(provider_message_id="", role=Role.ASSISTANT, text="same") for _ in range(2)], + attachments=[ + ParsedAttachment( + provider_attachment_id="ambiguous-drive-doc", + message_provider_id="", + message_position=0, + name="note.txt", + mime_type="text/plain", + ) + ], + ) + write_parsed_session_to_archive(conn, session) + conn.commit() + unowned = conn.execute( + "SELECT acquisition_status, ref_count FROM attachments WHERE display_name = 'note.txt'" + ).fetchone() + assert unowned is not None + assert tuple(unowned) == ("unfetched", 0) + assert conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0] == 0 + finally: + conn.close() + + report = verify_archive(tmp_path, checks=("blob-reference-closure", "attachment-coverage")) + + assert not report.blocking + closure = _check(report, "blob-reference-closure") + coverage = _check(report, "attachment-coverage") + assert closure.status is OutcomeStatus.OK, closure.summary + assert closure.evidence["acquired_attachment_missing_ref_count"] == 0 + assert coverage.status in {OutcomeStatus.OK, OutcomeStatus.SKIP}, coverage.summary + assert coverage.evidence.get("unreachable_count", 0) == 0 + + def test_attachment_blob_ref_joins_its_parent_raw_session(tmp_path: Path) -> None: _seed_coherent_archive(tmp_path) conn = _connect(tmp_path / "source.db") diff --git a/tests/unit/maintenance/test_blob_disposition_apply.py b/tests/unit/maintenance/test_blob_disposition_apply.py new file mode 100644 index 0000000000..9f85ad777b --- /dev/null +++ b/tests/unit/maintenance/test_blob_disposition_apply.py @@ -0,0 +1,426 @@ +"""Fault matrix for consuming an accepted blob disposition plan. + +The apply boundary is irreversible, so every test here names the mutation +that would make it red: deleting before restoring, trusting a stale plan, +accepting a changed source or denominator, or converting a blocked member +into a silent success. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +from polylogue.maintenance.blob_disposition import ( + BlobDisposition, + BlobDispositionContext, + BlobDispositionPlan, + build_disposition_context, + compile_disposition_plan, +) +from polylogue.maintenance.blob_disposition_apply import ( + MemberOutcome, + apply_disposition_plan, + restore_plan_members, + write_receipt, +) +from polylogue.sources.hooks import read_hook_spool_record +from polylogue.storage.blob_store import BlobStore + + +def _hook_envelope(event_id: str = "event-1", *, text: str = "ran a tool") -> dict[str, object]: + return { + "event_id": event_id, + "event_type": "PreToolUse", + "session_id": "session-1", + "timestamp": "2026-07-15T02:15:39Z", + "provider": "claude-code", + "payload": {"tool_name": "Bash", "detail": text}, + } + + +def _stored_bytes(envelope: dict[str, object], tmp_path: Path) -> bytes: + scratch = tmp_path / f"scratch-{envelope['event_id']}.json" + scratch.write_text(json.dumps(envelope, sort_keys=True), encoding="utf-8") + return json.dumps(read_hook_spool_record(scratch), ensure_ascii=False, sort_keys=True, indent=1).encode("utf-8") + + +def _write_spool_file(root: Path, envelope: dict[str, object]) -> Path: + target = root / "pending" / "2026-07-15" + target.mkdir(parents=True, exist_ok=True) + path = target / f"{envelope['event_id']}.json" + path.write_text(json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=4), encoding="utf-8") + return path + + +def _archive(tmp_path: Path) -> tuple[Path, Path, Path, Path]: + archive_root = tmp_path / "archive" + blob_root = archive_root / "blob" + blob_root.mkdir(parents=True) + hooks_root = archive_root / "hooks" + hooks_root.mkdir() + capture_spool = archive_root / "browser-capture" + capture_spool.mkdir() + source_db = archive_root / "source.db" + with sqlite3.connect(source_db) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB, ref_type TEXT)") + conn.execute( + "CREATE TABLE raw_sessions (raw_id TEXT, origin TEXT, native_id TEXT, blob_hash BLOB, " + "blob_size INTEGER, source_path TEXT, append_start_offset INTEGER)" + ) + with sqlite3.connect(archive_root / "index.db") as conn: + conn.execute("CREATE TABLE sessions (session_id TEXT)") + return archive_root, blob_root, hooks_root, capture_spool + + +def _plan_and_context( + archive_root: Path, + blob_root: Path, + *, + legacy_root: Path | None = None, + capture_spool: Path, +) -> tuple[BlobDispositionPlan, BlobDispositionContext]: + hook_sources = (("legacy-hook-spool-0", legacy_root),) if legacy_root is not None else () + context = build_disposition_context( + archive_root=archive_root, + blob_root=blob_root, + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + plan = compile_disposition_plan( + archive_root=archive_root, + blob_root=blob_root, + source_db=archive_root / "source.db", + context=context, + ) + return plan, context + + +def test_restoration_publishes_into_the_ordinary_spool_and_keeps_the_carrier(tmp_path: Path) -> None: + """Anti-vacuity: deleting the carrier during restoration makes this red.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + envelope = _hook_envelope("sole-copy") + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + assert plan.members[0].disposition is BlobDisposition.RESTORE_REQUIRED + + (result,) = restore_plan_members( + plan, + context=context, + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert result.outcome is MemberOutcome.RESTORED + restored = Path(result.detail) + assert restored.is_file() + assert read_hook_spool_record(restored) == json.loads(store.blob_path(blob_hash).read_bytes()) + assert store.blob_path(blob_hash).is_file() + + +def test_restoration_is_idempotent_by_logical_identity(tmp_path: Path) -> None: + """Anti-vacuity: matching only today's day shard double-delivers a retry. + + ``enqueue_hook_event`` refuses a collision inside the current day's shard + only, so a resident event spooled on any other day must be found by + identity or the retry writes a second carrier of the same event. + """ + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + envelope = _hook_envelope("sole-copy") + store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + + (first,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + # Relocate the restored carrier into another day's shard: the retry must + # still recognize it rather than publish a second copy. + relocated = hooks_root / "pending" / "2026-07-15" + relocated.mkdir(parents=True, exist_ok=True) + Path(first.detail).rename(relocated / "sole-copy.json") + + (second,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert first.outcome is MemberOutcome.RESTORED + assert second.outcome is MemberOutcome.RESTORATION_ALREADY_PRESENT + assert [path.name for path in hooks_root.rglob("*.json")] == ["sole-copy.json"] + + +def test_restoration_blocks_on_a_hostile_collision(tmp_path: Path) -> None: + """Anti-vacuity: overwriting on identity collision loses the resident event.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + store.write_from_bytes(_stored_bytes(_hook_envelope("collide", text="the stored call"), tmp_path)) + resident = hooks_root / "pending" / "2026-07-15" + resident.mkdir(parents=True) + (resident / "collide.json").write_text( + json.dumps(_hook_envelope("collide", text="a different call"), sort_keys=True), encoding="utf-8" + ) + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + + (result,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert result.outcome is MemberOutcome.BLOCKED + assert "different event" in result.detail + assert json.loads((resident / "collide.json").read_text())["payload"]["detail"] == "a different call" + + +def test_a_source_proof_appearing_after_planning_blocks_restoration(tmp_path: Path) -> None: + """Anti-vacuity: skipping revalidation restores material already at its source.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + legacy_root.mkdir() + store = BlobStore(blob_root) + envelope = _hook_envelope("late-arrival") + store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, _ = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + assert plan.members[0].disposition is BlobDisposition.RESTORE_REQUIRED + + _write_spool_file(legacy_root, envelope) + _, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + (result,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert result.outcome is MemberOutcome.BLOCKED + assert "no longer justified" in result.detail + + +def test_a_stale_authorized_digest_refuses_before_any_effect(tmp_path: Path) -> None: + """Anti-vacuity: applying without digest binding consumes an edited plan.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest="0" * 64, + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert not receipt.ok + assert any("does not match plan digest" in blocker for blocker in receipt.blockers) + assert store.blob_path(blob_hash).is_file() + + +def test_an_unresolved_member_refuses_the_whole_plan(tmp_path: Path) -> None: + """Anti-vacuity: applying a partially explained plan deletes beside a mystery.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + proven_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + mystery_hash, _ = store.write_from_bytes(b"%PDF-1.5\nunexplained\n") + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + assert plan.unresolved_count == 1 + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert not receipt.ok + assert any("not acceptable" in blocker for blocker in receipt.blockers) + assert store.blob_path(proven_hash).is_file() + assert store.blob_path(mystery_hash).is_file() + + +def test_an_active_writer_refuses_an_active_apply(tmp_path: Path) -> None: + """Anti-vacuity: unserialized apply races the archive's single writer.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + writer_block_reason="live pidfile PID 4242 is running", + dry_run=False, + ) + + assert not receipt.ok + assert any("writer is active" in blocker for blocker in receipt.blockers) + assert store.blob_path(blob_hash).is_file() + + +def test_a_changed_source_invalidates_the_member_before_deletion(tmp_path: Path) -> None: + """Anti-vacuity: trusting the planning-time proof deletes divergent material.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + spool_file = _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, _ = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + assert plan.accepted + + spool_file.write_text( + json.dumps(_hook_envelope("proven", text="rewritten at the source"), sort_keys=True), encoding="utf-8" + ) + _, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert not receipt.ok + assert store.blob_path(blob_hash).is_file() + assert any(result.outcome is MemberOutcome.BLOCKED for result in receipt.results) + + +def test_a_referenced_object_is_retained_not_deleted(tmp_path: Path) -> None: + """Anti-vacuity: deleting a proven-but-referenced object breaks a live row.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + with sqlite3.connect(archive_root / "source.db") as conn: + conn.execute("INSERT INTO blob_refs (blob_hash, ref_type) VALUES (?, ?)", (bytes.fromhex(blob_hash), "raw")) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert receipt.ok + assert [result.outcome for result in receipt.results] == [MemberOutcome.RETAINED_REFERENCED] + assert store.blob_path(blob_hash).is_file() + + +def test_a_dry_rehearsal_touches_nothing(tmp_path: Path) -> None: + """Anti-vacuity: a rehearsal that wrote would make the review meaningless.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + proven = _hook_envelope("proven") + _write_spool_file(legacy_root, proven) + store = BlobStore(blob_root) + proven_hash, _ = store.write_from_bytes(_stored_bytes(proven, tmp_path)) + sole_hash, _ = store.write_from_bytes(_stored_bytes(_hook_envelope("sole-copy"), tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=True, + ) + + assert receipt.ok and receipt.dry_run + assert store.blob_path(proven_hash).is_file() + assert store.blob_path(sole_hash).is_file() + assert list(hooks_root.rglob("*.json")) == [] + + +def test_receipt_totals_derive_from_member_outcomes(tmp_path: Path) -> None: + """Anti-vacuity: a summary counter maintained beside the members can drift.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + _write_spool_file(legacy_root, _hook_envelope("proven")) + store = BlobStore(blob_root) + store.write_from_bytes(_stored_bytes(_hook_envelope("proven"), tmp_path)) + store.write_from_bytes(_stored_bytes(_hook_envelope("sole-copy"), tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=True, + ) + + assert sum(receipt.counts.values()) == len(receipt.results) == len(plan.members) + destination = tmp_path / "receipts" / "disposition.json" + write_receipt(destination, receipt) + assert json.loads(destination.read_text())["counts"] == receipt.counts + + +def test_restoration_proceeds_while_other_members_are_unresolved(tmp_path: Path) -> None: + """Anti-vacuity: gating restoration on plan acceptance strands sole copies. + + Restoration never removes a carrier, so an unrelated unexplained object + must not delay preserving the only copy of wanted material. + """ + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + sole_hash, _ = store.write_from_bytes(_stored_bytes(_hook_envelope("sole-copy"), tmp_path)) + store.write_from_bytes(b"%PDF-1.5\nunexplained\n") + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + assert not plan.accepted + + results = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert [result.outcome for result in results] == [MemberOutcome.RESTORED] + assert store.blob_path(sole_hash).is_file() + + refused = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + assert not refused.ok diff --git a/tests/unit/maintenance/test_blob_disposition_plan.py b/tests/unit/maintenance/test_blob_disposition_plan.py new file mode 100644 index 0000000000..71cf3df995 --- /dev/null +++ b/tests/unit/maintenance/test_blob_disposition_plan.py @@ -0,0 +1,338 @@ +"""Laws for the physical blob disposition plan. + +Every test names the mutation that makes it red. The plan decides whether an +irreplaceable object is deleted, so the anti-vacuity conditions are all of the +same family: a prover that accepts material current sources do not hold, or a +classifier that converts "unknown" into "discard", must fail here. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest + +from polylogue.maintenance.blob_disposition import ( + AppendPrefixProver, + BlobDisposition, + BlobDispositionContext, + BlobDispositionError, + BlobDispositionPlan, + RawSourceCarrier, + RawSourceFileProver, + RestorationDestination, + SourceProofMode, + append_successors_by_hash, + build_disposition_context, + compile_disposition_plan, + raw_source_carriers_by_hash, + referenced_blob_hashes, +) +from polylogue.storage.blob_store import BlobStore + + +def _hook_envelope(event_id: str = "event-1", *, text: str = "ran a tool") -> dict[str, object]: + return { + "event_id": event_id, + "event_type": "PreToolUse", + "session_id": "session-1", + "timestamp": "2026-07-15T02:15:39Z", + "provider": "claude-code", + "payload": {"tool_name": "Bash", "detail": text}, + } + + +def _write_spool_file(root: Path, envelope: dict[str, object], *, indent: int | None = None) -> Path: + target = root / "pending" / "2026-07-15" + target.mkdir(parents=True, exist_ok=True) + path = target / f"{envelope['event_id']}.json" + path.write_text(json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=indent), encoding="utf-8") + return path + + +def _publish_blob(store: BlobStore, payload: bytes) -> str: + blob_hash, _ = store.write_from_bytes(payload) + return blob_hash + + +def _stored_envelope_bytes(spool_file: Path) -> bytes: + """Serialize the validated record the way acquisition stored it.""" + from polylogue.sources.hooks import read_hook_spool_record + + record = read_hook_spool_record(spool_file) + return json.dumps(record, ensure_ascii=False, sort_keys=True, indent=1).encode("utf-8") + + +def _empty_source_db(path: Path) -> Path: + with sqlite3.connect(path) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB, ref_type TEXT)") + conn.execute( + "CREATE TABLE raw_sessions (raw_id TEXT, origin TEXT, native_id TEXT, blob_hash BLOB, " + "blob_size INTEGER, source_path TEXT, append_start_offset INTEGER)" + ) + return path + + +def _context(tmp_path: Path, *, hook_roots: tuple[tuple[str, Path], ...] = ()) -> BlobDispositionContext: + blob_root = tmp_path / "blob" + blob_root.mkdir(exist_ok=True) + source_db = _empty_source_db(tmp_path / "source.db") + return build_disposition_context( + archive_root=tmp_path, + blob_root=blob_root, + source_db=source_db, + hook_spool_sources=hook_roots, + browser_capture_spool=tmp_path / "browser-capture", + ) + + +def test_hook_envelope_is_source_present_despite_differing_bytes(tmp_path: Path) -> None: + """Anti-vacuity: a byte-equality prover would call this a sole copy and delete it. + + Acquisition derives ``observed_at_ms`` and both sides serialize + independently, so the stored object never equals the spool file's bytes. + """ + spool_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope() + spool_file = _write_spool_file(spool_root, envelope, indent=4) + store = BlobStore(tmp_path / "blob") + blob_hash = _publish_blob(store, _stored_envelope_bytes(spool_file)) + assert store.blob_path(blob_hash).read_bytes() != spool_file.read_bytes() + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, + blob_root=store.root, + source_db=tmp_path / "source.db", + context=context, + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.SOURCE_PRESENT + assert member.proof is not None + assert member.proof.mode is SourceProofMode.SEMANTIC_EQUIVALENT + assert member.proof.source_path == str(spool_file) + assert plan.accepted + + +def test_hook_envelope_without_a_spool_file_is_restore_required(tmp_path: Path) -> None: + """Anti-vacuity: accepting an absent source would delete the only carrier.""" + spool_root = tmp_path / "legacy-hooks" + spool_root.mkdir() + envelope = _hook_envelope("orphan-event") + scratch = tmp_path / "scratch.json" + scratch.write_text(json.dumps(envelope, sort_keys=True), encoding="utf-8") + store = BlobStore(tmp_path / "blob") + _publish_blob(store, _stored_envelope_bytes(scratch)) + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.RESTORE_REQUIRED + assert member.restoration is not None + assert member.restoration.destination is RestorationDestination.HOOK_EVENT_SPOOL + assert member.restoration.logical_id == "orphan-event" + + +def test_same_event_id_with_different_content_is_not_a_source_proof(tmp_path: Path) -> None: + """Anti-vacuity: matching on identity alone would discard divergent material.""" + spool_root = tmp_path / "legacy-hooks" + _write_spool_file(spool_root, _hook_envelope(text="a completely different tool call")) + scratch = tmp_path / "scratch.json" + scratch.write_text(json.dumps(_hook_envelope(text="the stored call"), sort_keys=True), encoding="utf-8") + store = BlobStore(tmp_path / "blob") + _publish_blob(store, _stored_envelope_bytes(scratch)) + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.RESTORE_REQUIRED + + +def test_unclassifiable_material_is_unresolved_and_blocks_acceptance(tmp_path: Path) -> None: + """Anti-vacuity: routing unknown material to discard makes this green wrongly.""" + store = BlobStore(tmp_path / "blob") + _publish_blob(store, b"%PDF-1.5\nnot a session and not an envelope\n") + + context = _context(tmp_path) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.UNRESOLVED + assert plan.unresolved_count == 1 + assert not plan.accepted + + +def test_source_file_proof_requires_a_fresh_hash_not_path_existence(tmp_path: Path) -> None: + """Anti-vacuity: proving by path existence accepts a rewritten source.""" + source = tmp_path / "session.jsonl" + source.write_text('{"a": 1}\n', encoding="utf-8") + store = BlobStore(tmp_path / "blob") + blob_hash = _publish_blob(store, source.read_bytes()) + + prover = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source)),)}) + proof = prover.prove(blob_hash, store.blob_path(blob_hash), store.blob_path(blob_hash).stat().st_size) + assert proof is not None and proof.mode is SourceProofMode.BYTE_IDENTICAL + + source.write_text('{"a": 2}\n', encoding="utf-8") + assert prover.prove(blob_hash, store.blob_path(blob_hash), store.blob_path(blob_hash).stat().st_size) is None + + +def test_source_file_proof_accepts_an_exact_append_prefix(tmp_path: Path) -> None: + """Anti-vacuity: requiring whole-file equality would restore every append source.""" + store = BlobStore(tmp_path / "blob") + blob_hash = _publish_blob(store, b'{"a": 1}\n') + source = tmp_path / "session.jsonl" + source.write_bytes(b'{"a": 1}\n{"a": 2}\n') + + prover = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source)),)}) + proof = prover.prove(blob_hash, store.blob_path(blob_hash), 9) + assert proof is not None and proof.mode is SourceProofMode.STRICT_PREFIX + + source.write_bytes(b'{"z": 9}\n{"a": 2}\n') + assert prover.prove(blob_hash, store.blob_path(blob_hash), 9) is None + + +def test_append_prefix_only_supersedes_within_one_logical_item(tmp_path: Path) -> None: + """Anti-vacuity: an unscoped prefix search discards unrelated carriers.""" + store = BlobStore(tmp_path / "blob") + short = _publish_blob(store, b'{"a": 1}\n') + long = _publish_blob(store, b'{"a": 1}\n{"a": 2}\n') + + related = AppendPrefixProver({short: (long,)}, blob_store=store) + assert related.prove(short, store.blob_path(short), 9) is not None + + unrelated = AppendPrefixProver({}, blob_store=store) + assert unrelated.prove(short, store.blob_path(short), 9) is None + + +def test_append_successors_group_by_logical_identity(tmp_path: Path) -> None: + db = _empty_source_db(tmp_path / "source.db") + with sqlite3.connect(db) as conn: + conn.executemany( + "INSERT INTO raw_sessions (raw_id, origin, native_id, blob_hash, blob_size, source_path, " + "append_start_offset) VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + ("r1", "claude-code-session", "s1", bytes.fromhex("aa" * 32), 10, "/tmp/a", None), + ("r2", "claude-code-session", "s1", bytes.fromhex("bb" * 32), 20, "/tmp/a", None), + ("r3", "claude-code-session", "s2", bytes.fromhex("cc" * 32), 30, "/tmp/b", None), + ], + ) + successors = append_successors_by_hash(db) + assert successors == {"aa" * 32: ("bb" * 32,)} + assert raw_source_carriers_by_hash(db)["aa" * 32] == (RawSourceCarrier("/tmp/a"),) + + +def test_reference_union_covers_every_durable_relation(tmp_path: Path) -> None: + """Anti-vacuity: omitting one relation reports its blobs as unreferenced.""" + db = tmp_path / "source.db" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB)") + conn.execute("CREATE TABLE raw_sessions (blob_hash BLOB)") + conn.execute("CREATE TABLE raw_hook_events (blob_hash BLOB)") + conn.execute("CREATE TABLE raw_artifacts (blob_hash BLOB)") + conn.execute("CREATE TABLE blob_publication_reservations (blob_hash BLOB)") + for index, table in enumerate( + ("blob_refs", "raw_sessions", "raw_hook_events", "raw_artifacts", "blob_publication_reservations") + ): + conn.execute(f"INSERT INTO {table} (blob_hash) VALUES (?)", (bytes([index]) * 32,)) + + hashes = referenced_blob_hashes(db) + assert hashes == {bytes([index] * 32).hex() for index in range(5)} + + +def test_unreadable_reference_relation_fails_instead_of_reporting_zero(tmp_path: Path) -> None: + """Anti-vacuity: swallowing the error would license deleting the namespace.""" + db = tmp_path / "source.db" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB)") + conn.execute("CREATE VIEW raw_sessions AS SELECT blob_hash FROM missing_table") + + with pytest.raises(BlobDispositionError): + referenced_blob_hashes(db) + + +def test_plan_digest_binds_denominator_and_every_member(tmp_path: Path) -> None: + """Anti-vacuity: a digest over counts alone lets a member be swapped.""" + store = BlobStore(tmp_path / "blob") + _publish_blob(store, b"%PDF-1.5\nunexplained\n") + context = _context(tmp_path) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + reloaded = BlobDispositionPlan.from_dict(json.loads(json.dumps(plan.to_dict()))) + assert reloaded.digest() == plan.digest() + + mutated = BlobDispositionPlan.from_dict( + { + **plan.to_dict(), + "members": [{**plan.members[0].to_dict(), "disposition": BlobDisposition.SOURCE_PRESENT.value}], + } + ) + assert mutated.digest() != plan.digest() + + +def test_invalid_namespace_entries_block_acceptance(tmp_path: Path) -> None: + """Anti-vacuity: ignoring stray namespace entries hides unaccounted files.""" + blob_root = tmp_path / "blob" + (blob_root / "not-a-shard").mkdir(parents=True) + (blob_root / "not-a-shard" / "stray").write_bytes(b"x") + context = _context(tmp_path) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=blob_root, source_db=tmp_path / "source.db", context=context + ) + + assert plan.denominator.invalid_namespace_entries + assert not plan.accepted + + +def test_denominator_counts_the_complete_population(tmp_path: Path) -> None: + """Anti-vacuity: a sampled census would not reconcile against the walk.""" + spool_root = tmp_path / "legacy-hooks" + spool_file = _write_spool_file(spool_root, _hook_envelope("counted")) + store = BlobStore(tmp_path / "blob") + _publish_blob(store, _stored_envelope_bytes(spool_file)) + _publish_blob(store, b"%PDF-1.5\nunexplained\n") + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + assert plan.denominator.physical_file_count == 2 + assert plan.denominator.distinct_hash_count == 2 + assert sum(plan.counts.values()) == 2 + assert plan.counts[BlobDisposition.SOURCE_PRESENT.value] == 1 + assert plan.counts[BlobDisposition.UNRESOLVED.value] == 1 + + +def test_source_file_proof_accepts_the_recorded_append_span(tmp_path: Path) -> None: + """Anti-vacuity: without the recorded span, every increment-only row restores. + + An append-structured acquisition stores just its own increment, so the + object is neither the file nor the file's prefix; only ``file[start:]`` + reproduces it. + """ + store = BlobStore(tmp_path / "blob") + increment = b'{"a": 2}\n' + blob_hash = _publish_blob(store, increment) + source = tmp_path / "session.jsonl" + source.write_bytes(b'{"a": 1}\n' + increment) + + without_span = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source)),)}) + assert without_span.prove(blob_hash, store.blob_path(blob_hash), len(increment)) is None + + with_span = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source), 9),)}) + proof = with_span.prove(blob_hash, store.blob_path(blob_hash), len(increment)) + assert proof is not None and proof.mode is SourceProofMode.STRICT_PREFIX diff --git a/tests/unit/maintenance/test_embedding_preservation.py b/tests/unit/maintenance/test_embedding_preservation.py index ee1f3a6f50..968eed2c85 100644 --- a/tests/unit/maintenance/test_embedding_preservation.py +++ b/tests/unit/maintenance/test_embedding_preservation.py @@ -2,11 +2,18 @@ import json import sqlite3 +import subprocess +import sys +from contextlib import closing from pathlib import Path +from typing import Any import pytest +from polylogue.maintenance import embedding_preservation from polylogue.maintenance.embedding_preservation import ( + RestoreMissReason, + _receipt_path, delete_preserved_copy, preserve_embedding_vectors, restore_embedding_vectors, @@ -18,30 +25,50 @@ _HASH = b"h" * 32 _OTHER = b"o" * 32 _MISSING = b"m" * 32 +_VECTOR = b"\x00" * (1024 * 4) +_RECIPE = b"a" * 32 +_CONTRACT = b"b" * 32 -def _db(path: Path, *, vector: bytes = _HASH) -> None: - initialize_archive_database(path, ArchiveTier.EMBEDDINGS) +def _open(path: Path) -> sqlite3.Connection: conn = sqlite3.connect(path) loaded, error = try_load_sqlite_vec(conn) if not loaded: conn.close() pytest.skip(str(error)) - conn.execute( - "INSERT INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", - (vector.hex(), b"\x00" * (1024 * 4), "test"), - ) - conn.execute( - "INSERT INTO message_embeddings_meta (vector_derivation_hash, model, dimension, recipe_hash, output_contract_hash) " - "VALUES (?, 'test', 1024, ?, ?)", - (vector, b"a" * 32, b"b" * 32), - ) - conn.commit() - conn.close() + return conn + + +def _db(path: Path, *, vectors: tuple[bytes, ...] = (_HASH,), metadata_only: tuple[bytes, ...] = ()) -> None: + """Current-schema embeddings DB holding ``vectors`` plus vector-less metadata rows.""" + initialize_archive_database(path, ArchiveTier.EMBEDDINGS) + with closing(_open(path)) as conn: + for value in (*vectors, *metadata_only): + if value in vectors: + conn.execute( + "INSERT INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", + (value.hex(), _VECTOR, "test"), + ) + conn.execute( + "INSERT INTO message_embeddings_meta " + "(vector_derivation_hash, model, dimension, recipe_hash, output_contract_hash) " + "VALUES (?, 'test', 1024, ?, ?)", + (value, _RECIPE, _CONTRACT), + ) + conn.commit() -def _legacy_db(path: Path, *, vector: bytes = _HASH) -> None: - with sqlite3.connect(path) as conn: +def _legacy_db( + path: Path, + *, + vectors: tuple[bytes, ...] = (_HASH,), + output_contract_hash: bytes | None = _CONTRACT, +) -> None: + """Pre-v5 embeddings DB: hashes named ``embedding_input_hash``, identity columns nullable. + + Mirrors the live archive DDL at /realm/state/polylogue/embeddings.db. + """ + with closing(sqlite3.connect(path)) as conn: conn.executescript( """ CREATE TABLE message_embeddings_meta ( @@ -59,14 +86,27 @@ def _legacy_db(path: Path, *, vector: bytes = _HASH) -> None: ); """ ) - conn.execute( - "INSERT INTO message_embeddings VALUES (?, ?, 'test')", - (vector.hex(), b"\x00" * (1024 * 4)), - ) - conn.execute( - "INSERT INTO message_embeddings_meta VALUES (?, 'test', 1024, NULL, ?, ?)", - (vector, b"a" * 32, b"b" * 32), - ) + for value in vectors: + conn.execute("INSERT INTO message_embeddings VALUES (?, ?, 'test')", (value.hex(), _VECTOR)) + conn.execute( + "INSERT INTO message_embeddings_meta VALUES (?, 'test', 1024, NULL, ?, ?)", + (value, _RECIPE, output_contract_hash), + ) + conn.commit() + + +def _count(path: Path, table: str) -> int: + with closing(_open(path)) as conn: + return int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + + +def _ac2_proof(receipt: Any, path: Path, **override: Any) -> Path: + """Write the preservation receipt back as an AC2-passed deletion proof.""" + from dataclasses import asdict + + proof = asdict(receipt) | {"ac2_passed": True} | override + path.write_text(json.dumps(proof), encoding="utf-8") + return path def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: @@ -74,7 +114,7 @@ def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: preserved = tmp_path / "preserved.db" fresh = tmp_path / "fresh.db" _db(source) - _db(fresh, vector=_OTHER) + _db(fresh, vectors=(_OTHER,)) before = preserve_embedding_vectors(source, preserved) assert before.metadata_rows == before.vector_rows == 1 @@ -82,8 +122,8 @@ def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: restored = restore_embedding_vectors(fresh, preserved, {_HASH}) assert restored.restored_hashes == 1 - assert restored.missing_hashes == () - with sqlite3.connect(fresh) as conn: + assert restored.misses == () + with closing(sqlite3.connect(fresh)) as conn: assert ( conn.execute( "SELECT COUNT(*) FROM message_embeddings_meta WHERE vector_derivation_hash = ?", (_HASH,) @@ -91,8 +131,7 @@ def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: == 1 ) - proof = preserved.with_suffix(preserved.suffix + ".proof.json") - proof.write_text(json.dumps({"ac2_passed": True})) + proof = _ac2_proof(before, preserved.with_suffix(preserved.suffix + ".proof.json")) delete_preserved_copy(preserved, receipt_path=proof) assert not preserved.exists() assert not proof.exists() @@ -103,11 +142,13 @@ def test_missing_preserved_hash_is_enumerated(tmp_path: Path) -> None: preserved = tmp_path / "preserved.db" fresh = tmp_path / "fresh.db" _db(source) - _db(fresh, vector=_OTHER) + _db(fresh, vectors=(_OTHER,)) preserve_embedding_vectors(source, preserved) result = restore_embedding_vectors(fresh, preserved, {_HASH, _MISSING}) assert result.restored_hashes == 1 - assert result.missing_hashes == (_MISSING.hex(),) + assert [(miss.input_hash, miss.reason) for miss in result.misses] == [ + (_MISSING.hex(), RestoreMissReason.METADATA_ABSENT) + ] def test_restore_maps_legacy_embedding_input_hash_to_current_identity(tmp_path: Path) -> None: @@ -115,16 +156,233 @@ def test_restore_maps_legacy_embedding_input_hash_to_current_identity(tmp_path: preserved = tmp_path / "preserved.db" fresh = tmp_path / "fresh.db" _legacy_db(source) - _db(fresh, vector=_OTHER) + _db(fresh, vectors=(_OTHER,)) preserve_embedding_vectors(source, preserved) result = restore_embedding_vectors(fresh, preserved, {_HASH}) assert result.restored_hashes == 1 - with sqlite3.connect(fresh) as conn: + with closing(sqlite3.connect(fresh)) as conn: assert ( conn.execute( "SELECT COUNT(*) FROM message_embeddings_meta WHERE vector_derivation_hash = ?", (_HASH,) ).fetchone()[0] == 1 ) + + +def test_restore_batches_hashes_below_the_build_variable_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Red without batching: one IN list of six hashes exceeds a four-variable build.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + fresh = tmp_path / "fresh.db" + wanted = tuple(bytes([index]) * 32 for index in range(1, 7)) + _db(source, vectors=wanted) + _db(fresh, vectors=(_OTHER,)) + preserve_embedding_vectors(source, preserved) + + original = embedding_preservation._connect + + def small_limit(path: Any, *, readonly: bool) -> sqlite3.Connection: + conn = original(path, readonly=readonly) + if readonly: + conn.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 4) + return conn + + monkeypatch.setattr(embedding_preservation, "_connect", small_limit) + result = restore_embedding_vectors(fresh, preserved, set(wanted)) + + assert result.restored_hashes == len(wanted) + assert result.misses == () + + +def test_metadata_without_its_vector_is_a_miss_and_writes_nothing(tmp_path: Path) -> None: + """Red when metadata is written before its vector is found: a metadata row is the + tier's reuse signal, so one without a vector silently suppresses re-embedding.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + fresh = tmp_path / "fresh.db" + _db(source, vectors=(), metadata_only=(_HASH,)) + _db(fresh, vectors=(_OTHER,)) + preserve_embedding_vectors(source, preserved) + + result = restore_embedding_vectors(fresh, preserved, {_HASH}) + + assert result.restored_hashes == 0 + assert [(miss.input_hash, miss.reason) for miss in result.misses] == [ + (_HASH.hex(), RestoreMissReason.VECTOR_ABSENT) + ] + with closing(sqlite3.connect(fresh)) as conn: + assert ( + conn.execute( + "SELECT COUNT(*) FROM message_embeddings_meta WHERE vector_derivation_hash = ?", (_HASH,) + ).fetchone()[0] + == 0 + ) + + +def test_incomplete_legacy_metadata_is_a_typed_miss(tmp_path: Path) -> None: + """Red when an incomplete row is inserted: the current tier's NOT NULL identity + contract rejects it and aborts every remaining hash in the restore.""" + source = tmp_path / "legacy.db" + preserved = tmp_path / "preserved.db" + fresh = tmp_path / "fresh.db" + _legacy_db(source, vectors=(_HASH,), output_contract_hash=None) + _db(fresh, vectors=(_OTHER,)) + preserve_embedding_vectors(source, preserved) + + result = restore_embedding_vectors(fresh, preserved, {_HASH, _MISSING}) + + assert result.restored_hashes == 0 + assert [(miss.input_hash, miss.reason, miss.detail) for miss in result.misses] == [ + (_HASH.hex(), RestoreMissReason.METADATA_INCOMPLETE, "output_contract_hash"), + (_MISSING.hex(), RestoreMissReason.METADATA_ABSENT, ""), + ] + + +def test_deletion_refuses_a_receipt_naming_another_copy(tmp_path: Path) -> None: + """Red when the proof is not bound to the copy: any AC2-passed receipt authorizes + deleting any file.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + other = tmp_path / "other.db" + _db(source) + receipt = preserve_embedding_vectors(source, preserved) + other.write_bytes(preserved.read_bytes()) + + proof = _ac2_proof(receipt, tmp_path / "proof.json", copy=str(other)) + with pytest.raises(ValueError, match="different copy"): + delete_preserved_copy(preserved, receipt_path=proof) + assert preserved.exists() + + +def test_deletion_refuses_a_receipt_whose_digest_is_stale(tmp_path: Path) -> None: + """Red without a digest re-check: a copy mutated after its receipt still deletes.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + _db(source) + receipt = preserve_embedding_vectors(source, preserved) + with closing(_open(preserved)) as conn: + conn.execute("DELETE FROM message_embeddings WHERE vector_derivation_hash = ?", (_HASH.hex(),)) + conn.commit() + + proof = _ac2_proof(receipt, tmp_path / "proof.json") + with pytest.raises(ValueError, match="receipt digest"): + delete_preserved_copy(preserved, receipt_path=proof) + assert preserved.exists() + + +# Interrupting a backup from inside this process is impossible: sqlite3 discards +# exceptions raised in a progress callback, so the copy always runs to completion. +# The callback instead crashes the interpreter, which is the failure being guarded. +_CRASH_MID_BACKUP = """ +import os, sys +from pathlib import Path +from polylogue.maintenance import embedding_preservation as ep + +source, destination = Path(sys.argv[1]), Path(sys.argv[2]) +original = ep._connect + + +def interrupted(path, *, readonly): + conn = original(path, readonly=readonly) + if Path(path) != source: + return conn + + class Crash: + def __getattr__(self, name): + return getattr(conn, name) + + def __enter__(self): + conn.__enter__() + return self + + def __exit__(self, *exc): + return conn.__exit__(*exc) + + def backup(self, target, **kwargs): + conn.backup(target, pages=1, progress=lambda *_: os._exit(9)) + + return Crash() + + +ep._connect = interrupted +ep.preserve_embedding_vectors(source, destination) +""" + + +def test_a_crash_mid_backup_leaves_no_destination_file(tmp_path: Path) -> None: + """Red when the backup writes straight to the destination: the crash leaves a + truncated file there that no later run can tell apart from a whole copy.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + _db(source, vectors=tuple(bytes([index]) * 32 for index in range(1, 25))) + + crash = subprocess.run( + [sys.executable, "-c", _CRASH_MID_BACKUP, str(source), str(preserved)], + capture_output=True, + text=True, + ) + + assert crash.returncode == 9, crash.stderr + assert not preserved.exists() + assert not _receipt_path(preserved).exists() + + +def test_receipt_counts_come_from_the_completed_copy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Red when the receipt is read before the backup: a source written between the two + reads yields a receipt that describes no file.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + _db(source) + + original = embedding_preservation._table_digest + grew = False + + def growing_source(conn: sqlite3.Connection) -> Any: + nonlocal grew + result = original(conn) + if not grew: + grew = True + with closing(_open(source)) as writer: + writer.execute( + "INSERT INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", + (_OTHER.hex(), _VECTOR, "test"), + ) + writer.execute( + "INSERT INTO message_embeddings_meta " + "(vector_derivation_hash, model, dimension, recipe_hash, output_contract_hash) " + "VALUES (?, 'test', 1024, ?, ?)", + (_OTHER, _RECIPE, _CONTRACT), + ) + writer.commit() + return result + + monkeypatch.setattr(embedding_preservation, "_table_digest", growing_source) + receipt = preserve_embedding_vectors(source, preserved) + + assert receipt.metadata_rows == _count(preserved, "message_embeddings_meta") + assert receipt.vector_rows == _count(preserved, "message_embeddings") + with closing(embedding_preservation._connect(preserved, readonly=True)) as conn: + assert receipt.table_set_digest == original(conn)[0] + + +def test_preserved_copy_is_self_contained(tmp_path: Path) -> None: + """Red when the copy keeps the source's WAL mode: the rename moves only the main + file, leaving the copy beside sidecars that hold its content.""" + source = tmp_path / "source.db" + _db(source) + with closing(_open(source)) as conn: + conn.execute("PRAGMA journal_mode=WAL").fetchall() + vault = tmp_path / "vault" + preserved = vault / "preserved.db" + fresh = tmp_path / "fresh.db" + _db(fresh, vectors=(_OTHER,)) + + preserve_embedding_vectors(source, preserved) + + assert sorted(entry.name for entry in vault.iterdir()) == [ + "preserved.db", + "preserved.db.receipt.json", + ] + assert restore_embedding_vectors(fresh, preserved, {_HASH}).restored_hashes == 1 diff --git a/tests/unit/pipeline/test_archive_ingest_commit_batching.py b/tests/unit/pipeline/test_archive_ingest_commit_batching.py index ce142bbae6..1d37a2777b 100644 --- a/tests/unit/pipeline/test_archive_ingest_commit_batching.py +++ b/tests/unit/pipeline/test_archive_ingest_commit_batching.py @@ -506,6 +506,10 @@ def fake_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: # min(8, cpus-1), which is >= 2 on any real multi-core CI/dev host, so # an un-overridden call below would exercise the pool branch. monkeypatch.delenv("POLYLOGUE_INGEST_PARSE_WORKERS", raising=False) + # The walk-size tiering would send this synthetic corpus down the + # in-process branch on bytes alone; report a bulk-sized walk so the + # override, not the tier, is what this test measures. + monkeypatch.setattr(archive_ingest, "_submission_payload_bytes", lambda _submissions: 128 * 1024 * 1024) archive_root = workspace_env["archive_root"] sequential_sources = _build_sources(tmp_path, count=2, seed=97) @@ -513,7 +517,7 @@ def fake_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: assert pool_calls == [] # workers<=1 takes the sequential for-loop, no pool constructed assert result.counts["sessions"] == _expected_session_count(sequential_sources) - pooled_sources = _build_sources(tmp_path, count=2, seed=98) + pooled_sources = _build_sources(tmp_path, count=4, seed=98) result = asyncio.run(parse_sources_archive(archive_root, pooled_sources, parse_workers=3)) assert pool_calls == [3] # explicit override reaches the pool construction exactly assert result.counts["sessions"] == _expected_session_count(pooled_sources) @@ -554,6 +558,9 @@ def spying_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: monkeypatch.setattr(archive_ingest, "process_pool_executor", spying_process_pool_executor) monkeypatch.delenv("POLYLOGUE_INGEST_PARSE_WORKERS", raising=False) + # Report a bulk-sized walk so the dispatch tiering selects the pool this + # test exists to inspect (see _submission_payload_bytes). + monkeypatch.setattr(archive_ingest, "_submission_payload_bytes", lambda _submissions: 128 * 1024 * 1024) archive_root = workspace_env["archive_root"] sources = _build_sources(tmp_path, count=2, seed=113) @@ -562,3 +569,60 @@ def spying_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: assert calls == [2] assert result.counts["sessions"] == _expected_session_count(sources) assert not hasattr(archive_ingest, "ProcessPoolExecutor") + + +def _walk_bytes(sources: Sequence[Source]) -> int: + return sum(source.path.stat().st_size for source in sources if source.path is not None) + + +def test_small_walk_parses_in_process_with_identical_results( + tmp_path: Path, + workspace_env: dict[str, Path], + empty_archive_template: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An ordinary small walk parses in-process and lands exactly what the pool lands. + + ``resolve_archive_ingest_dispatch`` used to read CPU count alone, so a walk + of a handful of small files spawned one fresh interpreter per ambient + worker -- a full ``polylogue`` import each -- to parse them. Measured on a + 24-thread host, six one-file walks went 33-41 messages/s with the pool and + 401-424 messages/s without it, at 0.06 versus 0.84 process CPU utilization. + + Anti-vacuity, two ways: reverting the tiering makes the first arm construct + a pool and fail on ``pool_calls``; routing the in-process branch through + anything other than the same ``_parse_source_path_worker`` call makes the + two arms' session and message counts diverge. + """ + from polylogue.pipeline.services.process_pool import process_pool_executor as real_process_pool_executor + from tests.infra.archive_templates import clone_archive_template + + pool_calls: list[int] = [] + + def spying_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: + pool_calls.append(max_workers) + return real_process_pool_executor(max_workers=max_workers) + + monkeypatch.setattr(archive_ingest, "process_pool_executor", spying_process_pool_executor) + monkeypatch.delenv("POLYLOGUE_INGEST_PARSE_WORKERS", raising=False) + + in_process_root = workspace_env["archive_root"] + sources = _build_sources(tmp_path, count=3, seed=211) + assert _walk_bytes(sources) <= 8 * 1024 * 1024 # the tier the first arm relies on + + in_process = asyncio.run(parse_sources_archive(in_process_root, sources)) + assert pool_calls == [] + assert in_process.counts["sessions"] == _expected_session_count(sources) + + pooled_root = tmp_path / "pooled-archive" + clone_archive_template(empty_archive_template, pooled_root) + monkeypatch.setattr(archive_ingest, "_submission_payload_bytes", lambda _submissions: 128 * 1024 * 1024) + pooled = asyncio.run(parse_sources_archive(pooled_root, sources)) + # One pool, sized by min(path_count, cpus, ceiling) -- bounded by the walk + # rather than fixed, so this holds on a narrower host too. + assert len(pool_calls) == 1 + assert 2 <= pool_calls[0] <= len(sources) + + assert _counts(pooled_root / "index.db") == _counts(in_process_root / "index.db") + assert pooled.counts["sessions"] == in_process.counts["sessions"] + assert pooled.counts["messages"] == in_process.counts["messages"] diff --git a/tests/unit/pipeline/test_archive_ingest_shared_raw.py b/tests/unit/pipeline/test_archive_ingest_shared_raw.py index 67ee690960..ed5aa31b72 100644 --- a/tests/unit/pipeline/test_archive_ingest_shared_raw.py +++ b/tests/unit/pipeline/test_archive_ingest_shared_raw.py @@ -806,3 +806,82 @@ def require_source_transaction( assert len(_raw_rows_for_path(archive_root / "source.db", str(second_child))) == 1 with sqlite3.connect(f"file:{archive_root / 'index.db'}?mode=ro", uri=True) as conn: assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 4 + + +def _write_stem_identity_husk(root: Path, stem: str) -> Path: + """A JSON document under a Claude Code project whose only identity is its filename. + + The record shape satisfies dispatch's loose "looks like a message list" + admission but carries no record the Claude Code parser recognizes, so the + parse falls back to the discovery walk's ``fallback_id`` (the filename + stem) and yields a session with zero authored messages. + """ + path = root / ".claude" / "projects" / "proj" / "notes" / f"{stem}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({"messages": [{"role": "user", "content": "tool output shaped like a chat"}]}), + encoding="utf-8", + ) + return path + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stem", ["toolu_01ABCDEFGHIJKLMNOPQRSTUV", "wf_run_1"]) +async def test_archive_ingest_refuses_filename_stem_identity_without_authored_content( + tmp_path: Path, workspace_env: dict[str, Path], stem: str +) -> None: + """polylogue-b508: the one-shot importer must not mint fragment-identity husks. + + ``require_positive_conversational_evidence`` is the archive's admission law + for "parsed, but no conversation is present". Every other production write + path applies it -- the daemon decode worker, live batch convergence, the + incremental append route, and offline replay. ``parse_sources_archive`` + (reached from the public ``Polylogue.parse_file``/``parse_sources`` API and + the demo seeder) did not, so a JSON document that merely satisfies the + loose "has a messages list" shape became a session keyed on its own + filename stem with zero messages -- the ``toolu_*``/``wf_*`` fragment + phantom class this bead exists to make unrepresentable. (The ``*.meta`` + sibling shape is refused earlier, by its own declared artifact rule.) + + Anti-vacuity: dropping the ``write_pair`` evidence gate writes one session + row per parametrized stem, each ``provider_session_id`` equal to the stem + and ``COUNT(*) FROM messages`` zero. + """ + archive_root = workspace_env["archive_root"] + husk = _write_stem_identity_husk(tmp_path / "corpus", stem) + + result = await parse_sources_archive( + archive_root, + [Source(name="claude-code", path=husk)], + parse_workers=1, + ) + + assert result.parse_failures == 0 + assert result.counts.get("sessions", 0) == 0 + with sqlite3.connect(f"file:{archive_root / 'index.db'}?mode=ro", uri=True) as conn: + assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_archive_ingest_still_admits_a_real_session_through_the_same_gate( + tmp_path: Path, workspace_env: dict[str, Path] +) -> None: + """The evidence gate must not refuse a transcript that carries authored content. + + Pairs with the husk law above: the refusal is keyed on absent authored + content, not on the file's location or its identity shape. Anti-vacuity: + widening the gate to reject on anything the husk case has in common with + this one (same directory, same provider, same one-shot route) turns this + red. + """ + archive_root = workspace_env["archive_root"] + transcript = _write_session_shaped_workflow_journal(tmp_path / "sessions") + + result = await parse_sources_archive( + archive_root, + [Source(name="claude-code", path=transcript)], + parse_workers=1, + ) + + assert result.parse_failures == 0 + assert result.counts["sessions"] == 1 diff --git a/tests/unit/pipeline/test_message_identity_position_fallback.py b/tests/unit/pipeline/test_message_identity_position_fallback.py index a995a4101c..7d3404fa93 100644 --- a/tests/unit/pipeline/test_message_identity_position_fallback.py +++ b/tests/unit/pipeline/test_message_identity_position_fallback.py @@ -291,8 +291,8 @@ def _document_attachment(reference_id: str, position: int) -> ParsedAttachment: def test_duplicate_idless_turns_distinguished_by_referenced_document_own_their_attachments() -> None: """Block reference identity is content: turns citing different documents are different turns. - Anti-vacuity: dropping the block-reference discriminator from - ``message_owner_resolution`` makes both attachments raise + Anti-vacuity: dropping ``metadata`` from + ``_HASHED_FIELDS["ParsedContentBlock"]`` makes both attachments raise ``MessageOwnerAmbiguityError`` here, because the turns share role, timestamp, text, and block type. """ @@ -317,21 +317,6 @@ def test_duplicate_idless_turns_with_identical_document_reference_still_fail_clo session_revision_projection(_session(messages, [attachment])) -def test_block_reference_discriminator_leaves_unique_content_keys_unchanged() -> None: - """The reference tier only runs after the content tier collides.""" - timestamp = "2024-01-01T00:00:00Z" - first_references = [ - _document_only("doc-a", 0).model_copy(update={"text": "first", "timestamp": timestamp}), - _document_only("doc-b", 1).model_copy(update={"text": "second", "timestamp": timestamp}), - ] - other_references = [ - _document_only("doc-x", 0).model_copy(update={"text": "first", "timestamp": timestamp}), - _document_only("doc-y", 1).model_copy(update={"text": "second", "timestamp": timestamp}), - ] - - assert message_owner_resolution(first_references).keys == message_owner_resolution(other_references).keys - - def test_session_content_hash_degrades_ambiguous_attachment_to_unowned() -> None: """A parseable session must survive an attachment owner ambiguity. diff --git a/tests/unit/pipeline/test_pipeline_ids.py b/tests/unit/pipeline/test_pipeline_ids.py index d9e5a1f724..8f08c47795 100644 --- a/tests/unit/pipeline/test_pipeline_ids.py +++ b/tests/unit/pipeline/test_pipeline_ids.py @@ -251,7 +251,12 @@ def test_semantic_hash_partition_rejects_unclassified_and_duplicate_fields(monke def test_duplicate_idless_messages_with_only_position_difference_keep_owner_ambiguous() -> None: - """Position must not turn indistinguishable owner evidence into identity.""" + """Position must not turn indistinguishable owner evidence into identity. + + The strict revision projection is where ownership stays fail-closed; the + ingest content hash instead degrades the attachment to unowned so a + parseable session survives. + """ messages = [ _parsed_message("", "assistant", "repeat", "2024-01-01T00:00:00Z").model_copy(update={"position": position}) for position in (0, 1) @@ -264,12 +269,12 @@ def test_duplicate_idless_messages_with_only_position_difference_keep_owner_ambi mime_type="text/plain", ) + session = _parsed_session("s1", "title", messages, created_at=None, updated_at=None).model_copy( + update={"attachments": [attachment]} + ) + with pytest.raises(MessageOwnerAmbiguityError): - session_content_hash( - _parsed_session("s1", "title", messages, created_at=None, updated_at=None).model_copy( - update={"attachments": [attachment]} - ) - ) + session_revision_projection(session) @pytest.mark.parametrize( @@ -404,10 +409,10 @@ def test_session_revision_projection_golden_hashes() -> None: session = _golden_session() projection = session_revision_projection(session) - assert projection.session_hash.hex() == "87702d4073fdae9932d9b61f4399daa841c77528969645d29b8ac3d6b1134419" + assert projection.session_hash.hex() == "23d0b219777cf59e1b3b8fbe0a16f217e1f8129f9781c2dc4643e665102c4df7" assert [h.hex() for h in projection.message_hashes] == [ "bf3267d2bbb5b9f281401ca940a5a0f339174e750f6dd7b7a5aa70014b00640b", - "a7d0e29040820c1b0285c284a92aa1aad06aca56697aef3b45869f6a31a9bbf3", + "2518ce27da65142108dc3d78e65d7d360202814b6420d8644f50cff3cfe503c1", ] # Content-derived identity (message_id, name, mime_type) -- no longer a # hash of the provider attachment id (polylogue-aggz / polylogue-d8al): diff --git a/tests/unit/pipeline/test_process_pool.py b/tests/unit/pipeline/test_process_pool.py index c99148ae7c..00f48482a2 100644 --- a/tests/unit/pipeline/test_process_pool.py +++ b/tests/unit/pipeline/test_process_pool.py @@ -171,26 +171,49 @@ def test_parallel_threads_effective_treats_missing_probe_as_gil_enabled(monkeypa # failing first. -def test_resolve_archive_ingest_dispatch_defaults_to_resolve_parse_worker_count( +@pytest.mark.parametrize( + ("path_count", "total_bytes", "cpu_count", "expected_kind", "expected_workers"), + [ + # A single path never justifies a spawn, whatever it weighs. + (1, 512 * 1024 * 1024, 24, PoolKind.SEQUENTIAL, 1), + # Small-byte tier: in-process, matching resolve_ingest_batch_dispatch. + (400, 8 * 1024 * 1024, 24, PoolKind.SEQUENTIAL, 1), + # Mid tier: capped at 4 workers. + (400, 8 * 1024 * 1024 + 1, 24, PoolKind.PROCESS, 4), + (400, 64 * 1024 * 1024, 24, PoolKind.PROCESS, 4), + # Above the mid tier: min(path_count, cpus, ceiling). + (400, 64 * 1024 * 1024 + 1, 24, PoolKind.PROCESS, 16), + (3, 512 * 1024 * 1024, 24, PoolKind.PROCESS, 3), + (400, 512 * 1024 * 1024, 6, PoolKind.PROCESS, 6), + ], +) +def test_resolve_archive_ingest_dispatch_tiers_on_measured_walk_size( monkeypatch: pytest.MonkeyPatch, + path_count: int, + total_bytes: int, + cpu_count: int, + expected_kind: PoolKind, + expected_workers: int, ) -> None: - monkeypatch.setattr("polylogue.pipeline.services.process_pool.available_cpus", lambda **_: 9) - monkeypatch.setattr(sys, "_is_gil_enabled", lambda: True, raising=False) - plan = resolve_archive_ingest_dispatch() - assert plan.pool_kind is PoolKind.PROCESS - # GIL build: min(8, cpus-1) = min(8, 8) = 8. - assert plan.worker_count == 8 + """The plan is sized from the work the walk found, not from CPU count alone. + + Anti-vacuity: dropping either byte tier, or the ``path_count`` term in the + ``min``, changes at least one row here. Before the tiers existed every row + resolved to PROCESS with the ambient ceiling, so a one-file walk spawned 16 + interpreters to parse one file. + """ + monkeypatch.setattr("polylogue.pipeline.services.process_pool.available_cpus", lambda **_: cpu_count) + plan = resolve_archive_ingest_dispatch(path_count=path_count, total_bytes=total_bytes, worker_ceiling=16) + assert plan.pool_kind is expected_kind + assert plan.worker_count == expected_workers -def test_resolve_archive_ingest_dispatch_honors_explicit_override() -> None: - """``parse_workers`` (the demo seeder's force-sequential knob) wins over - the ambient CPU-based default, clamped to at least 1.""" - plan = resolve_archive_ingest_dispatch(parse_workers=1) +def test_resolve_archive_ingest_dispatch_honors_worker_ceiling(monkeypatch: pytest.MonkeyPatch) -> None: + """The caller's resolved ``POLYLOGUE_INGEST_PARSE_WORKERS`` ceiling still binds.""" + monkeypatch.setattr("polylogue.pipeline.services.process_pool.available_cpus", lambda **_: 24) + plan = resolve_archive_ingest_dispatch(path_count=400, total_bytes=512 * 1024 * 1024, worker_ceiling=2) assert plan.pool_kind is PoolKind.PROCESS - assert plan.worker_count == 1 - - plan_negative = resolve_archive_ingest_dispatch(parse_workers=-5) - assert plan_negative.worker_count == 1 + assert plan.worker_count == 2 @pytest.mark.parametrize( diff --git a/tests/unit/sources/test_parsers_drive.py b/tests/unit/sources/test_parsers_drive.py index 1e13e1f464..2da3772b72 100644 --- a/tests/unit/sources/test_parsers_drive.py +++ b/tests/unit/sources/test_parsers_drive.py @@ -802,12 +802,12 @@ def test_idless_document_only_turns_referencing_distinct_files_write_every_attac """AI Studio exports carry id-less, timestamp-less, text-less user turns whose only content is a Drive reference, one turn per file, all files sharing one display name. The turns differ only in the referenced file id, so the - attachment owner resolution must read block reference identity or every - such session fails parse (polylogue-prjai / polylogue-gmb3o). + attachment owner resolution must read the document block's ``metadata`` + or every such session fails parse (polylogue-prjai / polylogue-gmb3o). - Anti-vacuity: removing the block-reference tier from - ``message_owner_resolution`` makes ``session_content_hash`` raise - ``MessageOwnerAmbiguityError`` for this payload. + Anti-vacuity: dropping ``metadata`` from + ``_HASHED_FIELDS["ParsedContentBlock"]`` makes ``session_content_hash`` + raise ``MessageOwnerAmbiguityError`` for this payload. """ payload: JSONDocument = { "chunkedPrompt": { @@ -877,3 +877,53 @@ def inline_image(raw: bytes) -> JSONDocument: rows = conn.execute("SELECT message_id FROM attachment_refs").fetchall() assert len(rows) == 2 assert len({message_id for (message_id,) in rows}) == 2 + + +def test_same_timestamp_document_only_turns_keep_every_attachment_owned( + workspace_env: Mapping[str, Path], +) -> None: + """AI Studio stamps a run of id-less, text-less document turns with one shared + timestamp. Their role/timestamp revision anchor collides by construction, so + the cited Drive file is the only evidence separating them; without it the + attachment owner resolves onto an ambiguous key and the whole session fails + transform with ``attachment owner coordinate is indistinguishable from + another message`` (polylogue-prjai). + + Anti-vacuity: dropping ``metadata`` from + ``_HASHED_FIELDS["ParsedContentBlock"]`` makes + ``session_revision_projection`` raise ``MessageOwnerAmbiguityError`` at + coordinate ``(1, 0)`` and leaves every attachment unowned. + """ + stamp = "2026-04-07T02:20:20.469Z" + payload: JSONDocument = { + "chunkedPrompt": { + "chunks": [ + {"role": "user", "text": "Guidelines follow.", "createTime": "2026-04-06T19:29:05.295Z"}, + {"role": "user", "driveDocument": {"id": "file-a", "name": "chapter.md"}, "createTime": stamp}, + {"role": "user", "driveDocument": {"id": "file-b", "name": "chapter.md"}, "createTime": stamp}, + {"role": "user", "driveDocument": {"id": "file-c", "name": "chapter.md"}, "createTime": stamp}, + {"role": "user", "driveDocument": {"id": "file-d", "name": "chapter.md"}, "createTime": stamp}, + {"role": "model", "text": "Read.", "finishReason": "STOP", "createTime": stamp}, + ] + } + } + + result = parse_chunked_prompt("gemini", payload, "gemini-same-timestamp-documents") + assert [message.provider_message_id for message in result.messages] == [""] * 6 + assert {message.timestamp for message in result.messages[1:]} == {stamp} + assert [attachment.provider_attachment_id for attachment in result.attachments] == [ + "file-a", + "file-b", + "file-c", + "file-d", + ] + + # The strict projection is the path that has no ambiguous-owner tolerance. + session_revision_projection(result) + + db_path = db_setup(workspace_env) + with open_connection(db_path) as conn: + write_and_hydrate(PipelineRoundtrip(result, session_content_hash(result)), conn) + rows = conn.execute("SELECT message_id, attachment_id FROM attachment_refs ORDER BY message_id").fetchall() + assert len(rows) == 4 + assert len({message_id for message_id, _attachment_id in rows}) == 4 diff --git a/tests/unit/sources/test_tool_result_sidecars.py b/tests/unit/sources/test_tool_result_sidecars.py index e32480d6a1..f5ebc8259f 100644 --- a/tests/unit/sources/test_tool_result_sidecars.py +++ b/tests/unit/sources/test_tool_result_sidecars.py @@ -12,7 +12,8 @@ import os from pathlib import Path -from polylogue.core.enums import BlockType +from polylogue.config import Source +from polylogue.core.enums import BlockType, Provider from polylogue.sources.live.tool_result_sidecars import ( SidecarDebt, SidecarMatch, @@ -20,7 +21,10 @@ join_tool_result_sidecars_session_scoped, resolve_sibling_transcript_paths, ) +from polylogue.sources.origin_specs import artifact_rule_for_path from polylogue.sources.parsers.claude.code_parser import apply_tool_result_sidecars, parse_code +from polylogue.sources.revision_backfill import _parse_one +from polylogue.sources.source_parsing import iter_source_sessions_with_raw _TRUNCATED_NEEDLE = "zz_sentinel_needle_only_in_full_output" @@ -299,3 +303,85 @@ def test_session_scoped_join_never_emits_debt_for_subagent_meta_companion_files( assert result.matched == () assert result.debt == () + + +def test_tool_results_sidecar_never_becomes_a_session_on_either_chokepoint(tmp_path: Path) -> None: + """polylogue-b508: a ``tool-results/`` file is raw-only, whatever it contains. + + A tool call's own output can reproduce a genuine session-document shape -- + a messages list, even an ``id`` field shaped like the ``toolu_*`` id the + sidecar is named for. Content heuristics alone therefore cannot refuse this + family; the ``tool_result_sidecar`` path rule is the gate, and both parse + chokepoints must honour it: the discovery/acquisition walk that the one-shot + importer and the live watcher share, and the offline replay engine that + rebuilds from retained raws. + + Anti-vacuity: widening ``tool_result_sidecar``'s ``path_pattern`` so it no + longer matches, or relaxing its ``parse_policy`` from ``raw-only``, admits + a session whose ``provider_session_id`` is the ``toolu_*`` fragment id -- + the phantom shape this law forbids. + """ + body = json.dumps( + { + "id": "toolu_01ABCDEFGHIJKLMNOPQRSTUV", + "messages": [ + {"role": "user", "content": "tool output that happens to look like a chat"}, + {"role": "assistant", "content": "reply"}, + ], + } + ).encode("utf-8") + + sidecar = ( + tmp_path / ".claude" / "projects" / "proj" / "sess" / "tool-results" / "toolu_01ABCDEFGHIJKLMNOPQRSTUV.json" + ) + sidecar.parent.mkdir(parents=True) + sidecar.write_bytes(body) + + rule = artifact_rule_for_path(Provider.CLAUDE_CODE, str(sidecar)) + assert rule is not None + assert (rule.kind, rule.parse_policy) == ("tool_result_sidecar", "raw-only") + + replayed = _parse_one(Provider.CLAUDE_CODE, body, str(sidecar)) + assert replayed == [] + + acquired = list(iter_source_sessions_with_raw(Source(name="claude-code", path=sidecar), capture_raw=False)) + assert [session.provider_session_id for _raw, session in acquired] == [] + + +def test_sidecar_event_time_stays_unknown_when_the_file_carries_no_mtime_evidence() -> None: + """polylogue-x1gd: absent time evidence stays typed unknown, never invented. + + The sidecar file's own mtime is the only timestamp evidence this join has -- + sidecars carry no embedded time, and for genuine debt the owning + ``tool_result`` block is by definition unresolvable. When ``stat`` cannot + supply it (``file_mtime_ms`` stays ``None``), the emitted + ``claude_tool_result_sidecar`` event must carry no timestamp, leaving + ``occurred_at_ms`` NULL, rather than an ingestion-time stamp that would + read downstream as "this sidecar was written at import". + + Anti-vacuity: defaulting the missing-mtime branch to the current clock or + to the transcript's own timestamp makes both ``event.timestamp`` values + non-None and turns this red. + """ + from polylogue.sources.live.tool_result_sidecars import SidecarJoinResult + + payload = [_record("m-aaa", "toolu_AAA", "small full text")] + join_result = SidecarJoinResult( + matched=( + SidecarMatch( + tool_use_id="toolu_AAA", + filename="toolu_AAA.txt", + byte_size=15, + content_hash="0" * 64, + was_truncated=False, + full_text="small full text", + ), + ), + debt=(SidecarDebt(filename="orphan123.txt", byte_size=7, reason="no_owning_tool_result_block"),), + ) + + acquired = parse_code(payload, "fallback-sidecar", tool_result_sidecars=join_result) + + sidecar_events = [event for event in acquired.session_events if event.event_type == "claude_tool_result_sidecar"] + assert len(sidecar_events) == 2 + assert [event.timestamp for event in sidecar_events] == [None, None] diff --git a/tests/unit/storage/test_archive_tiers_write.py b/tests/unit/storage/test_archive_tiers_write.py index 5ea576daaa..34e3c91ee0 100644 --- a/tests/unit/storage/test_archive_tiers_write.py +++ b/tests/unit/storage/test_archive_tiers_write.py @@ -3066,7 +3066,7 @@ def test_refresh_thread_reorder_only_touches_changed_span(tmp_path: Path) -> Non thread_row = conn.execute( "SELECT session_count, depth FROM threads WHERE thread_id = ?", (root_session_id,) ).fetchone() - assert dict(thread_row) == {"session_count": 6, "depth": 5} + assert dict(thread_row) == {"session_count": 6, "depth": 0} assert not any( "thread_sessions" in stmt and stmt.lstrip().startswith(("INSERT", "UPDATE", "DELETE")) for stmt in statements @@ -5642,3 +5642,56 @@ def test_real_writer_persists_current_semantic_fingerprints_on_replay(tmp_path: assert tuple(replay) == (*tuple(first), "raw-replay") finally: conn.close() + + +def test_fresh_archive_reads_back_attachment_provenance_through_the_envelope(tmp_path: Path) -> None: + """A freshly bootstrapped index round-trips attachment direction and producer. + + The envelope reader selects ``attachment_refs.direction``/``producer_ref`` + unconditionally, so a fresh schema that omits either column fails at + statement preparation for every session, with or without attachments. + + Anti-vacuity: drop the ``direction`` and ``producer_ref`` columns from + ``ATTACHMENT_REFS_SPEC`` and this fails -- the write raises ``table + attachment_refs has no column named direction`` and the envelope read + raises ``no such column: r.direction``. + """ + conn = _connect(tmp_path / "index.db") + try: + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="attachment-envelope-provenance", + messages=[ + ParsedMessage(provider_message_id="m1", role=Role.USER, text="here is my file"), + ParsedMessage(provider_message_id="m2", role=Role.ASSISTANT, text="here is the chart"), + ], + attachments=[ + ParsedAttachment( + provider_attachment_id="a1", + message_provider_id="m1", + name="input.txt", + mime_type="text/plain", + ), + ParsedAttachment( + provider_attachment_id="a2", + message_provider_id="m2", + name="chart.png", + mime_type="image/png", + ), + ], + ) + + session_id = write_parsed_session_to_archive(conn, session) + envelope = read_archive_session_envelope(conn, session_id) + + provenance = { + attachment.display_name: (attachment.direction, attachment.producer_ref) + for message in envelope.messages + for attachment in message.attachments + } + assert provenance["input.txt"] == ("user_input", None) + model_direction, model_producer = provenance["chart.png"] + assert model_direction == "model_output" + assert model_producer is not None + finally: + conn.close() diff --git a/tests/unit/storage/test_attachment_reacquisition.py b/tests/unit/storage/test_attachment_reacquisition.py new file mode 100644 index 0000000000..9fc6e9816f --- /dev/null +++ b/tests/unit/storage/test_attachment_reacquisition.py @@ -0,0 +1,134 @@ +"""Attachment reacquisition across capture revisions (polylogue-4zqh3). + +An upload-only claude.ai ``files`` reference records a name and a size but no +bytes, so its first ingest is honestly ``unfetched``. When a later revision of +the same capture carries the payload as ``extracted_content``, the bytes must +land as an ``acquired`` blob under the *same* attachment identity — a second +identity would strand the original reference and double-count the attachment. + +Anti-vacuity: drop ``extracted_content`` from the ``files`` branch of +``attachment_from_meta`` and the second revision stays ``unfetched``; fold +acquisition state into ``_attachment_id`` and the two revisions mint different +identities, growing the reference count. +""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from pathlib import Path + +import pytest + +from polylogue.sources.parsers.base import ParsedSession +from polylogue.sources.parsers.claude import parse_ai +from polylogue.storage.blob_store import BlobStore +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.archive_tiers.write import write_parsed_session_to_archive + +SESSION_UUID = "reacquisition-session" +FILE_UUID = "upload-only-file" +PAYLOAD = "restored attachment payload\n" +PAYLOAD_BYTES = PAYLOAD.encode("utf-8") + + +def _capture(*, extracted_content: str | None) -> dict[str, object]: + """One claude.ai capture revision carrying a single upload-only reference.""" + file_record: dict[str, object] = { + "file_uuid": FILE_UUID, + "uuid": FILE_UUID, + "file_kind": "blob", + "file_name": "restored.md", + "size_bytes": len(PAYLOAD_BYTES), + "path": "/mnt/user-data/uploads/restored.md", + "success": True, + } + if extracted_content is not None: + file_record["extracted_content"] = extracted_content + return { + "uuid": SESSION_UUID, + "name": "Attachment reacquisition", + "chat_messages": [ + { + "uuid": "m0", + "sender": "human", + "text": "Please read this.", + "files": [file_record], + } + ], + } + + +def _connect(path: Path) -> sqlite3.Connection: + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + initialize_archive_tier(conn, ArchiveTier.INDEX) + return conn + + +def _preacquired(store: BlobStore, session: ParsedSession) -> dict[int, tuple[bytes | None, int, str]]: + acquired: dict[int, tuple[bytes | None, int, str]] = {} + for attachment in session.attachments: + if attachment.inline_bytes is None: + continue + blob_hash, size = store.write_from_bytes(attachment.inline_bytes) + acquired[id(attachment)] = (bytes.fromhex(blob_hash), size, "acquired") + return acquired + + +def _attachment_state(conn: sqlite3.Connection) -> sqlite3.Row: + row: sqlite3.Row | None = conn.execute( + "SELECT attachment_id, display_name, byte_count, blob_hash, acquisition_status FROM attachments" + ).fetchone() + assert row is not None + return row + + +def _ref_count(conn: sqlite3.Connection) -> int: + return int(conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0]) + + +def test_upload_only_reference_gains_bytes_at_a_stable_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + store = BlobStore(tmp_path / "blob") + monkeypatch.setattr("polylogue.storage.blob_store.get_blob_store", lambda: store) + conn = _connect(tmp_path / "index.db") + + before = parse_ai(_capture(extracted_content=None), "fallback") + write_parsed_session_to_archive(conn, before, preacquired_attachment_blobs=_preacquired(store, before)) + + unfetched = _attachment_state(conn) + assert unfetched["acquisition_status"] == "unfetched" + assert unfetched["blob_hash"] is None + assert unfetched["byte_count"] == len(PAYLOAD_BYTES) + identity = str(unfetched["attachment_id"]) + assert _ref_count(conn) == 1 + + after = parse_ai(_capture(extracted_content=PAYLOAD), "fallback") + write_parsed_session_to_archive(conn, after, preacquired_attachment_blobs=_preacquired(store, after)) + + acquired = _attachment_state(conn) + assert str(acquired["attachment_id"]) == identity + assert acquired["acquisition_status"] == "acquired" + assert bytes(acquired["blob_hash"]) == hashlib.sha256(PAYLOAD_BYTES).digest() + assert acquired["byte_count"] == len(PAYLOAD_BYTES) + assert store.read_all(hashlib.sha256(PAYLOAD_BYTES).hexdigest()) == PAYLOAD_BYTES + assert _ref_count(conn) == 1 + + +def test_replaying_the_acquired_revision_changes_nothing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + store = BlobStore(tmp_path / "blob") + monkeypatch.setattr("polylogue.storage.blob_store.get_blob_store", lambda: store) + conn = _connect(tmp_path / "index.db") + + for _ in range(2): + session = parse_ai(_capture(extracted_content=PAYLOAD), "fallback") + write_parsed_session_to_archive(conn, session, preacquired_attachment_blobs=_preacquired(store, session)) + + acquired = _attachment_state(conn) + assert acquired["acquisition_status"] == "acquired" + assert bytes(acquired["blob_hash"]) == hashlib.sha256(PAYLOAD_BYTES).digest() + assert _ref_count(conn) == 1 diff --git a/tests/unit/storage/test_attachment_relink.py b/tests/unit/storage/test_attachment_relink.py index 0f56666080..75c8cc0c26 100644 --- a/tests/unit/storage/test_attachment_relink.py +++ b/tests/unit/storage/test_attachment_relink.py @@ -15,8 +15,9 @@ import pytest -from polylogue.core.enums import Provider -from polylogue.pipeline.services.ingest_worker import ingest_record +from polylogue.core.enums import Provider, Role +from polylogue.pipeline.services.ingest_worker import IngestRecordResult, SessionWritePayload, ingest_record +from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession from polylogue.storage.attachment_relink import ( UnrecoverableAttachmentReason, plan_orphaned_attachment_relink, @@ -304,3 +305,79 @@ def test_plan_is_dry_run_by_default_makes_no_writes(tmp_path: Path) -> None: "SELECT 1 FROM attachment_refs WHERE attachment_id = ?", (attachment_id,) ).fetchone() assert still_orphaned is None + + +def test_owner_ambiguous_orphan_is_reported_typed_not_raised(tmp_path: Path) -> None: + """A ref-less attachment whose owner is ambiguous must not abort the plan. + + ``_write_attachments`` retains an attachment claimed by two indistinguishable + messages as a typed unowned row with no ``attachment_refs`` edge, so the row + is an orphan by ``_read_orphaned_attachment_ids``' definition and the raw + re-parse scan reaches it. Re-parsing reproduces the same ambiguity, and + ``attachment_message_owner_key`` raises for it. + + Anti-vacuity: remove the ``except MessageOwnerAmbiguityError`` handler in + ``_match_session_payload`` and this fails with + ``MessageOwnerAmbiguityError: attachment owner coordinate is + indistinguishable from another message`` instead of returning a plan -- + the same traceback that aborts ``polylogue ops maintenance + blob-reference-closure``. + """ + blob_store = BlobStore(tmp_path / "blob") + index_conn = _index_conn(tmp_path / "index.db") + source_conn = _source_conn(tmp_path / "source.db") + + session = ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="ambiguous-attachment-owner", + messages=[ParsedMessage(provider_message_id="", role=Role.ASSISTANT, text="same") for _ in range(2)], + attachments=[ + ParsedAttachment( + provider_attachment_id="ambiguous-drive-doc", + message_provider_id="", + message_position=0, + name="note.txt", + mime_type="text/plain", + ) + ], + ) + session_id = write_parsed_session_to_archive(index_conn, session) + index_conn.commit() + orphan_row = index_conn.execute("SELECT attachment_id FROM attachments WHERE display_name = 'note.txt'").fetchone() + assert orphan_row is not None + assert index_conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0] == 0 + + _write_raw_row(source_conn, blob_store, _CLAUDE_AI_PAYLOAD, raw_id="raw-1", source_path="conversations.json") + + def _parser(record: RawSessionRecord) -> IngestRecordResult: + """Stand in for the raw re-parse: the raw still holds the ambiguity.""" + return IngestRecordResult( + raw_id=record.raw_id, + sessions=[SessionWritePayload(session_id=session_id, content_hash="0" * 64, parsed_session=session)], + ) + + plan = plan_orphaned_attachment_relink( + index_conn, + source_conn, + archive_root=tmp_path, + blob_root=blob_store.root, + raw_session_parser=_parser, + ) + + assert plan.orphan_count == 1 + assert plan.eligible == () + assert plan.unrecoverable_samples[0].attachment_id == str(orphan_row["attachment_id"]) + assert plan.unrecoverable_samples[0].reason_kind is UnrecoverableAttachmentReason.OWNER_AMBIGUOUS + assert "more than one message claims its owner coordinate" in plan.unrecoverable_samples[0].reason + + exec_result = relink_orphaned_attachments( + index_conn, + source_conn, + archive_root=tmp_path, + blob_root=blob_store.root, + dry_run=False, + raw_session_parser=_parser, + ) + index_conn.commit() + assert exec_result.relinked_count == 0 + assert index_conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0] == 0