From 83f4cdcabc41b826da4fda95909c49cdd3e6d5db Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 16:49:50 +0200 Subject: [PATCH 01/47] fix(agentctl): keep full corpus out of lane verification --- .agentctl/project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agentctl/project.toml b/.agentctl/project.toml index 73fd7e8865..c14545aab6 100644 --- a/.agentctl/project.toml +++ b/.agentctl/project.toml @@ -23,7 +23,7 @@ root = "/realm/worktrees" default_base = "origin/master" identity_check = ["git", "diff", "--quiet", "--", "pyproject.toml", "uv.lock"] checkpoint_untracked = true -verification_operations = ["verify_affected", "verify_quick", "verify_all"] +verification_operations = ["verify_quick", "verify_affected"] [conflicts] exact_files = ["pyproject.toml", "uv.lock"] From e19bfbe21a35f065f2f8f06cada1b05f61bda7ab Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 17:12:28 +0200 Subject: [PATCH 02/47] fix(agentctl): let affected verification await pytest slot --- .agentctl/project.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.agentctl/project.toml b/.agentctl/project.toml index c14545aab6..ce1af43e54 100644 --- a/.agentctl/project.toml +++ b/.agentctl/project.toml @@ -37,7 +37,9 @@ verification-graph = ["devtools/verify.py", "pyproject.toml"] [operations.verify_affected] description = "Run Polylogue's affected-test verification plan" exec = ["env", "POLYLOGUE_PYTEST_WORKERS=2", "devtools", "verify"] -pool = "pytest" +# The verifier queues its actual pytest run into the single-worker host slot. +# Keep the outer verifier out of that same pool so it can wait for the slot. +pool = "normal" result = "pytest" cache = "tree+environment" timeout_seconds = 3600 From 71b51a43e02d745da523aec07581fe02da5dd043 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:58:30 +0200 Subject: [PATCH 03/47] fix(agentctl): leave affected verification to GitHub --- .agentctl/project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agentctl/project.toml b/.agentctl/project.toml index ce1af43e54..1838fb9a2e 100644 --- a/.agentctl/project.toml +++ b/.agentctl/project.toml @@ -23,7 +23,7 @@ root = "/realm/worktrees" default_base = "origin/master" identity_check = ["git", "diff", "--quiet", "--", "pyproject.toml", "uv.lock"] checkpoint_untracked = true -verification_operations = ["verify_quick", "verify_affected"] +verification_operations = ["verify_quick"] [conflicts] exact_files = ["pyproject.toml", "uv.lock"] From cec8e0e8699046e802b261156bcb60f19dcb0808 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:03:57 +0200 Subject: [PATCH 04/47] fix(devtools): scope descriptor-only verification --- devtools/verify.py | 94 +++++++++++++++++++++++++----- tests/unit/devtools/test_verify.py | 62 ++++++++++++++++++++ 2 files changed, 142 insertions(+), 14 deletions(-) diff --git a/devtools/verify.py b/devtools/verify.py index 49d20afab2..a38cd10fdc 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -80,6 +80,17 @@ #: overlap without multiplying cache churn or exhausting the pytest cgroup. CORPUS_MAX_WORKERS = 2 _AGENTCTL_OPERATION_ARGV = {"verify_affected": (), "verify_quick": ("--quick",), "verify_all": ("--all",)} +_PROJECT_DESCRIPTOR = ".agentctl/project.toml" +# These tests read the AgentCTL descriptor directly. They are the bounded +# contract for a descriptor-only change; Python changes still use Testmon. +DESCRIPTOR_CONTRACT_TESTS = ( + "tests/unit/devtools/test_deployment_browser_smoke_service.py::test_declared_browser_smoke_has_no_private_browser_service_lease", + "tests/unit/devtools/test_deployment_browser_smoke_service.py::test_declared_live_provider_proof_declares_no_port_lease", + "tests/unit/devtools/test_deployment_browser_smoke_service.py::test_sinnixd_parser_accepts_the_unleased_shared_chrome_operation", + "tests/unit/devtools/test_dev_loop_service.py::test_declared_operation_has_a_json_contract_and_no_retired_keys", + "tests/unit/devtools/test_seeded_archive_cache_gc.py::test_declared_agentctl_operation_is_bounded_and_previewable", + "tests/unit/devtools/test_verify.py::test_verify_quick_descriptor_accepts_the_declared_json_projection", +) _UNMEASURED_WORKLOAD_DIMENSIONS = ( "cpu_ms", "current_rss_bytes", @@ -153,7 +164,8 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: def _pytest_steps(*, selection: str, worker_args: Sequence[str]) -> list[tuple[str, list[str]]]: """Build one complete collection, or an affected collection with tracing.""" - testmon = selection != "all" + testmon = selection not in {"all", "descriptor"} + collection_args = CLOSED_WORLD_COLLECTION_ARGS[:-1] if selection == "descriptor" else CLOSED_WORLD_COLLECTION_ARGS command = [ venv_python(root=ROOT), "-m", @@ -169,11 +181,12 @@ def _pytest_steps(*, selection: str, worker_args: Sequence[str]) -> list[tuple[s "-p", PROGRESS_PLUGIN_NAME, *managed_plugin_args(testmon=testmon), - *CLOSED_WORLD_COLLECTION_ARGS, + *collection_args, *(["--testmon", f"--testmon-env={TESTMON_ENVIRONMENT}", "--testmon-forceselect"] if testmon else []), "-p", "no:randomly", *worker_args, + *(DESCRIPTOR_CONTRACT_TESTS if selection == "descriptor" else []), # Never under pytest-cov: testmon owns the tracer, and refuses to share # it with branch coverage. ] @@ -192,6 +205,53 @@ def build_verify_steps(*, quick: bool, selection: str = "all") -> list[tuple[str return steps +def _git_changed_paths(root: Path) -> frozenset[str] | None: + """Return committed and working-tree paths, or ``None`` if Git is unavailable.""" + try: + base = None + for candidate in ("origin/master", "master", "HEAD^"): + resolved = subprocess.run( + ["git", "rev-parse", "--verify", candidate], + cwd=root, + capture_output=True, + text=True, + check=False, + timeout=10, + ) + if resolved.returncode == 0 and resolved.stdout.strip(): + base = resolved.stdout.strip() + break + if base is None: + return None + paths: set[str] = set() + for command in ( + ["git", "diff", "--name-only", "--no-ext-diff", f"{base}...HEAD", "--"], + ["git", "diff", "--name-only", "--no-ext-diff", "HEAD", "--"], + ["git", "ls-files", "--others", "--exclude-standard"], + ): + result = subprocess.run( + command, + cwd=root, + capture_output=True, + text=True, + check=False, + timeout=10, + ) + if result.returncode != 0: + return None + paths.update(line for line in result.stdout.splitlines() if line) + return frozenset(paths) + except (OSError, subprocess.TimeoutExpired): + return None + + +def _selection_for_changes(changed_paths: frozenset[str] | None) -> str: + """Choose bounded descriptor checks only for an exact descriptor diff.""" + if changed_paths == frozenset({_PROJECT_DESCRIPTOR}): + return "descriptor" + return "affected" + + def _normalize_managed_pytest_environment(env: dict[str, str]) -> None: env.pop("PYTEST_ADDOPTS", None) env.pop("PYTEST_PLUGINS", None) @@ -522,7 +582,7 @@ def _early_gate_failure_result(started: float, metadata: Mapping[str, Any]) -> d def _scope(*, quick: bool, selection: str) -> VerificationScope: if quick: return VerificationScope.NON_TEST - return VerificationScope.AFFECTED if selection == "affected" else VerificationScope.COMPLETE + return VerificationScope.AFFECTED if selection in {"affected", "descriptor"} else VerificationScope.COMPLETE def _emit(payload: Mapping[str, Any], *, use_json: bool, operation: str | None) -> None: @@ -584,6 +644,7 @@ def _finish_interrupted_verification( started: float, scope: VerificationScope, args: argparse.Namespace, + selection: str, agentctl_operation: str | None, exit_code: int, termination_reason: str, @@ -602,7 +663,7 @@ def _finish_interrupted_verification( verification_scope=scope.value, final_git_head=git_head(ROOT), pytest_aggregate={ - "selection_mode": "quick" if args.quick else "all" if args.all_tests else "affected", + "selection_mode": "quick" if args.quick else selection, "outcomes": {}, "terminal_green": False, "complete_corpus_covered": False, @@ -712,14 +773,16 @@ def _main(argv: list[str] | None = None, *, agentctl_operation: str | None = Non _anchor_verification_paths() validate_authority_matrix() started = time.monotonic() + selection = "all" if args.all_tests else "affected" + if not args.quick and not args.all_tests: + selection = _selection_for_changes(_git_changed_paths(ROOT)) seeded_from_primary = sync_testmon_graph(ROOT) graph = inspect_testmon_graph(ROOT) - if graph.status is TestmonGraphStatus.UNUSABLE: + if graph.status is TestmonGraphStatus.UNUSABLE and selection != "descriptor": # An unusable lane copy cannot be an authority. If the primary seed # was unavailable, discard it so this run honestly reseeds. discard_testmon_graph(ROOT) graph = inspect_testmon_graph(ROOT) - selection = "all" if args.all_tests else "affected" scope = _scope(quick=args.quick, selection=selection) try: assert_polylogue_matches_checkout(ROOT, context="devtools verify") @@ -749,13 +812,13 @@ def _main(argv: list[str] | None = None, *, agentctl_operation: str | None = Non selection_mode=selection, graph_status=str(graph.status), graph_reason=graph.reason, - full_rerun_cause=graph.full_rerun_cause, + full_rerun_cause=graph.full_rerun_cause if selection != "descriptor" else None, seed_source=str(testmon_datafile(primary_worktree())) if seeded_from_primary else None, seed_source_mtime_ns=( testmon_datafile(primary_worktree()).stat().st_mtime_ns if seeded_from_primary else None ), ) - if graph.status is TestmonGraphStatus.UNUSABLE: + if graph.status is TestmonGraphStatus.UNUSABLE and selection != "descriptor": payload = _finish_and_record_verification( run=run, exit_code=2, @@ -767,12 +830,13 @@ def _main(argv: list[str] | None = None, *, agentctl_operation: str | None = Non sys.stderr.write(f"verify: {graph.reason}; no usable primary seed was available.\n") _emit(payload, use_json=args.json, operation=agentctl_operation) return 2 - if graph.status is TestmonGraphStatus.ABSENT: - sys.stderr.write("verify: no testmon datafile: this run seeds it and runs every test.\n") - elif graph.full_rerun_cause: - sys.stderr.write( - f"verify: {graph.full_rerun_cause} since the graph was written: this run re-executes every test.\n" - ) + if selection != "descriptor": + if graph.status is TestmonGraphStatus.ABSENT: + sys.stderr.write("verify: no testmon datafile: this run seeds it and runs every test.\n") + elif graph.full_rerun_cause: + sys.stderr.write( + f"verify: {graph.full_rerun_cause} since the graph was written: this run re-executes every test.\n" + ) steps = build_verify_steps(quick=args.quick, selection=selection) try: results: list[dict[str, Any]] = [] @@ -795,6 +859,7 @@ def _main(argv: list[str] | None = None, *, agentctl_operation: str | None = Non started=started, scope=scope, args=args, + selection=selection, agentctl_operation=agentctl_operation, exit_code=128 + exc.signum, termination_reason=signal.Signals(exc.signum).name.lower(), @@ -805,6 +870,7 @@ def _main(argv: list[str] | None = None, *, agentctl_operation: str | None = Non started=started, scope=scope, args=args, + selection=selection, agentctl_operation=agentctl_operation, exit_code=130, termination_reason="operator_interrupt", diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index c139072ad2..8206c6f2ac 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -295,6 +295,68 @@ def test_verify_quick_descriptor_accepts_the_declared_json_projection() -> None: assert projection["operation"] == "verify_quick" +def test_descriptor_only_changes_use_contract_tests_and_python_changes_use_testmon() -> None: + """A descriptor-only diff is bounded to descriptor contracts. + + Anti-vacuity: selecting the affected mode for the descriptor or selecting + descriptor contracts for a Python diff makes one of the boundary checks + below fail. + """ + assert verify._selection_for_changes(frozenset({".agentctl/project.toml"})) == "descriptor" + assert verify._selection_for_changes(frozenset({"polylogue/example.py"})) == "affected" + assert verify._selection_for_changes(frozenset({".agentctl/project.toml", "polylogue/example.py"})) == "affected" + assert verify._selection_for_changes(None) == "affected" + + descriptor_command = verify._pytest_steps(selection="descriptor", worker_args=[])[0][1] + assert "--testmon" not in descriptor_command + assert "tests" not in descriptor_command + assert descriptor_command[-len(verify.DESCRIPTOR_CONTRACT_TESTS) :] == list(verify.DESCRIPTOR_CONTRACT_TESTS) + + affected_command = verify._pytest_steps(selection="affected", worker_args=[])[0][1] + assert "--testmon" in affected_command + assert "--testmon-forceselect" in affected_command + assert not any(nodeid in affected_command for nodeid in verify.DESCRIPTOR_CONTRACT_TESTS) + + +def test_verify_main_routes_descriptor_diff_to_bounded_selection( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The default verifier route applies the descriptor boundary.""" + from devtools.agent_env import AGENT_PRINCIPAL, AGENT_PRINCIPAL_ENV + + captured: dict[str, Any] = {} + history: dict[str, Any] = {} + monkeypatch.setenv(AGENT_PRINCIPAL_ENV, AGENT_PRINCIPAL) + monkeypatch.setattr(verify, "ROOT", tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(verify, "_git_changed_paths", lambda _root: frozenset({".agentctl/project.toml"})) + monkeypatch.setattr(verify, "sync_testmon_graph", lambda _root: False) + monkeypatch.setattr( + verify, + "inspect_testmon_graph", + lambda _root: SimpleNamespace( + status=verify.TestmonGraphStatus.USABLE, + reason="testmon datafile present", + full_rerun_cause="the installed packages changed", + ), + ) + monkeypatch.setattr(verify, "assert_polylogue_matches_checkout", lambda *_args, **_kwargs: None) + monkeypatch.setattr(verify, "git_head", lambda _root: "head") + monkeypatch.setattr( + verify, + "build_verify_steps", + lambda **kwargs: captured.update(kwargs) or [("gate lint", ["true"])], + ) + monkeypatch.setattr(verify, "_run", lambda *_args, **_kwargs: (0, 0.1, {"diagnosis": "gate_passed"})) + monkeypatch.setattr(verify, "append_verify_history", lambda payload: history.update(payload)) + monkeypatch.setattr(verify, "append_verification_evidence", lambda _payload: None) + monkeypatch.setattr(verify, "prune_successful_verify_runs", lambda **_kwargs: None) + + assert verify._main([]) == 0 + assert captured["selection"] == "descriptor" + assert history["pytest_aggregate"]["selection_mode"] == "descriptor" + + def test_pytest_receipt_decodes_report_and_selection(tmp_path: Path) -> None: run = VerifyRun(tier="test", argv=[], git_head="head", root=tmp_path) artifacts = run.start_step(label="pytest focused", cmd=[sys.executable, "-m", "pytest"]) From 070cbc8fe16afeafb3d37938e0ac84856ddfd1f8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:05:31 +0200 Subject: [PATCH 05/47] fix(devtools): satisfy verifier test typing --- tests/unit/devtools/test_verify.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 8206c6f2ac..1fcc1f967d 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -14,7 +14,15 @@ import pytest import tomllib -from devtools import agent_env, gate, required_gate, verify, verify_runs, why +from devtools import ( + agent_env, + gate, + required_gate, + verify, + verify_runs, + why, +) +from devtools.testmon_provision import TestmonGraphStatus from devtools.verification_result import declared_verification_result from devtools.verify_runs import ( CURRENT_RUN_PATH, @@ -335,17 +343,22 @@ def test_verify_main_routes_descriptor_diff_to_bounded_selection( verify, "inspect_testmon_graph", lambda _root: SimpleNamespace( - status=verify.TestmonGraphStatus.USABLE, + status=TestmonGraphStatus.USABLE, reason="testmon datafile present", full_rerun_cause="the installed packages changed", ), ) monkeypatch.setattr(verify, "assert_polylogue_matches_checkout", lambda *_args, **_kwargs: None) monkeypatch.setattr(verify, "git_head", lambda _root: "head") + + def capture_steps(**kwargs: Any) -> list[tuple[str, list[str]]]: + captured.update(kwargs) + return [("gate lint", ["true"])] + monkeypatch.setattr( verify, "build_verify_steps", - lambda **kwargs: captured.update(kwargs) or [("gate lint", ["true"])], + capture_steps, ) monkeypatch.setattr(verify, "_run", lambda *_args, **_kwargs: (0, 0.1, {"diagnosis": "gate_passed"})) monkeypatch.setattr(verify, "append_verify_history", lambda payload: history.update(payload)) From d25a2f1603c53d2af296eba07b1dc31e9cc5b6fd Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:13:49 +0200 Subject: [PATCH 06/47] test(devtools): isolate verifier routing contract --- tests/unit/devtools/test_verify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 1fcc1f967d..92e8ae4e79 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -335,6 +335,7 @@ def test_verify_main_routes_descriptor_diff_to_bounded_selection( captured: dict[str, Any] = {} history: dict[str, Any] = {} monkeypatch.setenv(AGENT_PRINCIPAL_ENV, AGENT_PRINCIPAL) + monkeypatch.setattr(verify, "refuse_verify_tier", lambda _argv, _env: None) monkeypatch.setattr(verify, "ROOT", tmp_path) monkeypatch.chdir(tmp_path) monkeypatch.setattr(verify, "_git_changed_paths", lambda _root: frozenset({".agentctl/project.toml"})) From 424a2f22f6803ac15878dbc1605ec80b09635225 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 19:58:35 +0200 Subject: [PATCH 07/47] fix(agentctl): keep manual affected verification in pytest pool --- .agentctl/project.toml | 4 +--- tests/unit/devtools/test_verify.py | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.agentctl/project.toml b/.agentctl/project.toml index 1838fb9a2e..3f181e4ef2 100644 --- a/.agentctl/project.toml +++ b/.agentctl/project.toml @@ -37,9 +37,7 @@ verification-graph = ["devtools/verify.py", "pyproject.toml"] [operations.verify_affected] description = "Run Polylogue's affected-test verification plan" exec = ["env", "POLYLOGUE_PYTEST_WORKERS=2", "devtools", "verify"] -# The verifier queues its actual pytest run into the single-worker host slot. -# Keep the outer verifier out of that same pool so it can wait for the slot. -pool = "normal" +pool = "pytest" result = "pytest" cache = "tree+environment" timeout_seconds = 3600 diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 92e8ae4e79..9fc942a2d9 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -292,13 +292,21 @@ def test_verify_quick_descriptor_accepts_the_declared_json_projection() -> None: descriptor = tomllib.loads((verify.ROOT / ".agentctl/project.toml").read_text(encoding="utf-8")) operation = descriptor["operations"]["verify_quick"] + affected = descriptor["operations"]["verify_affected"] + complete = descriptor["operations"]["verify_all"] projection = declared_verification_result( {"exit_code": 0, "status": "success", "verification_scope": "non-test"}, operation="verify_quick", ) + assert descriptor["workspace"]["verification_operations"] == ["verify_quick"] assert operation["exec"] == ["devtools", "verify", "--quick"] assert operation["result"] == "json" + assert affected["exec"] == ["env", "POLYLOGUE_PYTEST_WORKERS=2", "devtools", "verify"] + assert affected["pool"] == "pytest" + assert affected["result"] == "pytest" + assert complete["exec"] == ["env", "POLYLOGUE_PYTEST_WORKERS=2", "devtools", "verify", "--all"] + assert complete["pool"] == "pytest" assert projection["kind"] == "polylogue.verification-result" assert projection["operation"] == "verify_quick" From 423e44a10967ff362f066b394f74c5df8b1de5cc Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:47:09 +0200 Subject: [PATCH 08/47] fix: Run hosted verification pytest inside the workflow job A GitHub Actions workflow job is its own serialisation domain: one self-hosted runner executes one job at a time, under the job plane's slice. It runs pytest in place instead of queueing through the workstation's pytest slot, which required the workstation's queue runner on the runner's PATH. Co-Authored-By: Claude Opus 5 --- .github/workflows/verify.yml | 4 +- CLAUDE.md | 9 ++-- TESTING.md | 4 ++ devtools/pytest_slot.py | 62 ++++++++++++++++++++----- tests/unit/devtools/test_agent_env.py | 19 ++++++++ tests/unit/devtools/test_pytest_slot.py | 47 ++++++++++++++++++- 6 files changed, 127 insertions(+), 18 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 75d3ed5f0c..143eeb0776 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -2,8 +2,8 @@ name: Verify # The real test evidence for a change. Hosted CI runs only the quick gate, so a # green PR check there does not mean tests ran; this job runs `devtools verify` -# on the workstation runner, where pytest goes through the host's single -# `pytest` pueue slot automatically. +# on the workstation runner. One runner executes one job at a time, so pytest +# runs in the job rather than through the workstation's pytest pueue slot. on: pull_request: merge_group: diff --git a/CLAUDE.md b/CLAUDE.md index ee82e2d8f0..8d25c32850 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,10 +148,11 @@ explicit-and-retryable or a typed permanent refusal. a lane with SQLite backup, replacing unusable lane copies. If no seed is available, the run reports a full seed run; `--all` runs every test and updates fingerprints, and `--quick` is the static gates alone. -- Every managed pytest run holds the host's single `pytest` pueue slot. Only - the pytest worker's `POLYLOGUE_PYTEST_SLOT=held` marker authorizes direct - execution. Every other caller queues, waits, reads the captured log the run - prints, and refuses if pueued is unreachable. +- Every managed pytest run holds the host's single `pytest` pueue slot. Direct + execution is authorized by the pytest worker's `POLYLOGUE_PYTEST_SLOT=held` + marker or by a GitHub Actions workflow job, which the single self-hosted + runner already serialises. Every other caller queues, waits, reads the + captured log the run prints, and refuses if pueued is unreachable. - `devtools why` — explain the last run before reading receipts by hand. - `devtools gate ` — one named invariant check (`gate --list`); `verify --quick` is the fast subset. `status`, `render [|all] diff --git a/TESTING.md b/TESTING.md index f22835afd1..0287469af3 100644 --- a/TESTING.md +++ b/TESTING.md @@ -41,6 +41,10 @@ share one workstation, so `devtools test` and the pytest step of - The pytest-group runner sets `POLYLOGUE_PYTEST_SLOT=held` for the task that owns the slot, so pytest runs in place and streams its output as before. +- A GitHub Actions workflow job (`GITHUB_ACTIONS=true` with a `GITHUB_RUN_ID`) + runs pytest in place too. One self-hosted runner executes one job at a time + under the job plane's slice, so hosted verification is already serialised and + never depends on the workstation's queue tooling being on the runner's PATH. - Every other caller, including a generic AgentCTL job, is queued as `pueue add --group pytest --label polylogue:{test,verify}:`, and the command waits for the task, reports the exit code pueue recorded, and prints the captured log path under diff --git a/devtools/pytest_slot.py b/devtools/pytest_slot.py index a5a0aee77f..d7fd648941 100644 --- a/devtools/pytest_slot.py +++ b/devtools/pytest_slot.py @@ -5,11 +5,14 @@ load control the daemon applies to its own jobs, so concurrent runs contend for the same cores and disk until a long job passes its timeout. -Every managed pytest run therefore holds the host's `pytest` pueue group (one -task at a time). A run already inside the pytest queue task holds the slot -already, marked explicitly with ``POLYLOGUE_PYTEST_SLOT=held``. Generic -Sinnixd job identity does not imply pytest-slot ownership: lane jobs queue and -wait here. +Every managed pytest run started from a session or a lane therefore holds the +host's `pytest` pueue group (one task at a time). A run already inside the +pytest queue task holds the slot already, marked explicitly with +``POLYLOGUE_PYTEST_SLOT=held``. Generic Sinnixd job identity does not imply +pytest-slot ownership: lane jobs queue and wait here. + +A GitHub Actions workflow job is the other execution context that already runs +one at a time, and it runs pytest in place; see :func:`inside_workflow_job`. pueue 4 records the full client environment of ``pueue add`` into a user-only state file, so the adder runs with a reduced environment and the managed pytest @@ -47,6 +50,7 @@ "basetemp_root", "contained_pytest_run", "holds_pytest_slot", + "inside_workflow_job", "main", "remove_temp_tree", "run_pytest", @@ -60,6 +64,14 @@ #: Explicit escape, for the hermetic test of this mechanism. SLOT_ESCAPE_ENV: Final = "POLYLOGUE_PYTEST_SLOT" SLOT_HELD: Final = "held" +#: What the receipt records for a run that executed here as a workflow job. +SLOT_WORKFLOW: Final = "github workflow job" + +#: GitHub Actions sets both in every workflow job. The run id is what +#: distinguishes a job from a shell that merely exported the flag. +WORKFLOW_MARKER_ENV: Final = "GITHUB_ACTIONS" +WORKFLOW_MARKER_VALUE: Final = "true" +WORKFLOW_RUN_ENV: Final = "GITHUB_RUN_ID" #: The host group whose parallelism is one. PYTEST_GROUP: Final = "pytest" @@ -90,7 +102,8 @@ class PytestSlotUnavailableError(RuntimeError): @dataclass(frozen=True) class SlotOutcome: returncode: int - #: What the receipt records: ``pueue task 12`` or ``held``. + #: What the receipt records: ``pueue task 12``, ``held``, or the + #: context that ran pytest in place. slot: str #: Where the queued run's output landed, or None when it streamed. log_path: Path | None = None @@ -181,6 +194,31 @@ def holds_pytest_slot(env: Mapping[str, str]) -> bool: return env.get(SLOT_ESCAPE_ENV) == SLOT_HELD or inside_declared_pytest_worker(env) +def inside_workflow_job(env: Mapping[str, str]) -> bool: + """Whether this process is a GitHub Actions workflow job. + + A workflow job is its own serialisation domain: the repository registers a + single self-hosted runner, which executes one job at a time, and the verify + workflow cancels a superseded run for the same ref. It also runs under the + job plane's slice, so it is inside the load control the pytest slot exists + to give a session subagent. + + Queueing from here would instead make hosted verification depend on the + workstation's private queue installation being on the runner's PATH, and + would leave a queued task to outlive its waiter when GitHub cancels a job. + """ + return env.get(WORKFLOW_MARKER_ENV) == WORKFLOW_MARKER_VALUE and bool(env.get(WORKFLOW_RUN_ENV)) + + +def _runs_pytest_in_place(env: Mapping[str, str]) -> str | None: + """The receipt value for a caller that runs pytest here, or None to queue.""" + if holds_pytest_slot(env): + return SLOT_HELD + if inside_workflow_job(env): + return SLOT_WORKFLOW + return None + + def adder_environment(env: Mapping[str, str]) -> dict[str, str]: """The reduced environment the ``pueue add`` client runs with.""" return {key: env[key] for key in INHERITED_ENVIRONMENT_KEYS if env.get(key)} @@ -456,14 +494,16 @@ def run_pytest( """Run a managed pytest command, acquiring the host's pytest slot first. When the pytest-group runner sets ``POLYLOGUE_PYTEST_SLOT=held``, the slot - is already held and the command runs here, streaming as before. Every other - caller is queued in the host's single-slot ``pytest`` group and its output - is captured. + is already held and the command runs here, streaming as before; a GitHub + Actions workflow job runs here for the reason :func:`inside_workflow_job` + gives. Every other caller is queued in the host's single-slot ``pytest`` + group and its output is captured. """ argv, contained, scratch = contained_pytest_run(command, env=env, root=root) - if holds_pytest_slot(env): + in_place = _runs_pytest_in_place(env) + if in_place is not None: completed = subprocess.run(argv, cwd=cwd, env=contained, stdout=stdout, stderr=stdout) - outcome = SlotOutcome(returncode=completed.returncode, slot=SLOT_HELD) + outcome = SlotOutcome(returncode=completed.returncode, slot=in_place) else: outcome = _queue(argv, cwd=cwd, env=contained, root=root, label=label) # A failed run keeps its scratch: that is when the leftovers are worth diff --git a/tests/unit/devtools/test_agent_env.py b/tests/unit/devtools/test_agent_env.py index 67c89ab803..fabe3849f6 100644 --- a/tests/unit/devtools/test_agent_env.py +++ b/tests/unit/devtools/test_agent_env.py @@ -13,6 +13,7 @@ "sinnixd-pueue-pytest-verify_affected-job.scope\n" ) OUTSIDE_CGROUP = "0::/user.slice/user-1000.slice/user@1000.service/app.slice/shell.scope\n" +WORKFLOW_RUNNER_CGROUP = "0::/sinnixd.slice/sinnixd-work.slice/github-runner-polylogue.service\n" def outside_cgroup() -> str: @@ -27,6 +28,10 @@ def deployed_pytest_cgroup() -> str: return DEPLOYED_PYTEST_CGROUP +def workflow_runner_cgroup() -> str: + return WORKFLOW_RUNNER_CGROUP + + def test_outside_agent_jobs_nothing_changes() -> None: assert agent_env.agent_worker_cap(8, {}, cgroup_reader=outside_cgroup) == 8 assert agent_env.refuse_verify_tier([], {}, cgroup_reader=outside_cgroup) is None @@ -85,3 +90,17 @@ def test_non_pytest_queue_worker_remains_agent_bound() -> None: assert not agent_env.inside_declared_pytest_worker(environment, cgroup_reader=deployed_agent_cgroup) assert agent_env.refuse_bare_pytest(environment, cgroup_reader=deployed_agent_cgroup) is not None + + +def test_the_workflow_runner_is_not_an_agent_job() -> None: + """Hosted verification runs the test tier; widening the agent slices would refuse it. + + Anti-vacuity: adding the runner's ``sinnixd-work.slice`` to the agent slices + makes every assertion here red. + """ + environment = {"GITHUB_ACTIONS": "true", "GITHUB_RUN_ID": "33895265963", "CI": "true"} + + assert not agent_env.inside_agent_job(environment, cgroup_reader=workflow_runner_cgroup) + assert agent_env.refuse_verify_tier([], environment, cgroup_reader=workflow_runner_cgroup) is None + assert agent_env.refuse_bare_pytest(environment, cgroup_reader=workflow_runner_cgroup) is None + assert agent_env.agent_worker_cap(8, environment, cgroup_reader=workflow_runner_cgroup) == 8 diff --git a/tests/unit/devtools/test_pytest_slot.py b/tests/unit/devtools/test_pytest_slot.py index 277027fd9d..a6b63343af 100644 --- a/tests/unit/devtools/test_pytest_slot.py +++ b/tests/unit/devtools/test_pytest_slot.py @@ -6,7 +6,10 @@ ``test_the_adder_environment_carries_only_the_allowed_keys`` red. Dropping either half of the temporary-directory containment (the ``--basetemp`` argument or the exported TMPDIR) makes -``test_a_queued_run_contains_its_temporary_trees`` red. +``test_a_queued_run_contains_its_temporary_trees`` red. Removing the workflow +branch of ``devtools.pytest_slot._runs_pytest_in_place`` makes +``test_a_workflow_job_runs_without_the_local_queue_tools`` red -- the run +refuses on the absent queue runner instead of executing. """ from __future__ import annotations @@ -27,9 +30,11 @@ from devtools import cloud_sentinels, pytest_slot from devtools.pytest_slot import ( BASETEMP_ROOT_ENV, + SLOT_WORKFLOW, PytestSlotUnavailableError, basetemp_root, holds_pytest_slot, + inside_workflow_job, run_pytest, ) @@ -219,6 +224,46 @@ def test_explicit_slot_holder_runs_directly(tmp_path: Path, monkeypatch: pytest. assert _calls(record) == [], "a slot holder must not talk to pueue" +def test_a_workflow_job_runs_without_the_local_queue_tools(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Hosted verification must not depend on the workstation's queue installation.""" + empty = tmp_path / "empty" + empty.mkdir() + monkeypatch.setenv("PATH", str(empty)) + marker = tmp_path / "pytest-ran" + workflow = _environment(PATH=str(empty), GITHUB_ACTIONS="true", GITHUB_RUN_ID="33895265963") + + assert not holds_pytest_slot(workflow), "a workflow job does not own the host slot" + assert inside_workflow_job(workflow) + outcome = run_pytest( + _marker_command(marker), + cwd=str(tmp_path), + env=workflow, + root=tmp_path, + label="polylogue:verify:1", + ) + + assert marker.exists(), "the workflow job did not run pytest" + assert (outcome.returncode, outcome.slot) == (0, SLOT_WORKFLOW) + + +def test_a_workflow_flag_without_a_run_identity_still_queues(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The exported flag alone is not a workflow job.""" + record = _install_fake_pueue(tmp_path, monkeypatch) + marker = tmp_path / "pytest-ran" + + outcome = run_pytest( + _marker_command(marker), + cwd=str(tmp_path), + env=_environment(GITHUB_ACTIONS="true"), + root=tmp_path, + label="polylogue:verify:1", + ) + + assert not marker.exists() + assert outcome.slot == "pueue task 7" + assert _calls(record)[0]["argv"][0] == "add" + + def test_an_agent_pool_worker_queues_its_focused_run_in_the_pytest_pool( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From ea41272e72428c16ba488e0a22ae64c4f1b99d8c Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 19:49:57 +0200 Subject: [PATCH 09/47] fix: Classify every pytest-pool worker and hermetically test the slot `verify_all` declares `pool = "pytest"` but was absent from the operation fallback, so its worker re-queued into the single-slot group it already occupied whenever the runner did not export `SINNIXD_QUEUE_POOL`. The declarations are now the test's own fixture, so a new pytest-pool operation cannot drift out of the classifier. The slot tests also read whatever the workstation had deployed: the declared worker's assertion held only because the run sat inside the deployed pytest cgroup, and queueing resolved the real `sinnixd-queue-run` from PATH. The cgroup is stubbed and both queue tools are fakes that are the whole PATH, so the unreachable-queue test now reaches the pueue check it is named for. Co-Authored-By: Claude Opus 5 --- devtools/agent_env.py | 7 ++- tests/unit/devtools/cgroups.py | 53 +++++++++++++++++ tests/unit/devtools/test_agent_env.py | 64 ++++++++++++--------- tests/unit/devtools/test_pytest_slot.py | 76 +++++++++++++++++++------ 4 files changed, 154 insertions(+), 46 deletions(-) create mode 100644 tests/unit/devtools/cgroups.py diff --git a/devtools/agent_env.py b/devtools/agent_env.py index 8476089f52..03d964e574 100644 --- a/devtools/agent_env.py +++ b/devtools/agent_env.py @@ -20,7 +20,12 @@ QUEUE_POOL_ENV = "SINNIXD_QUEUE_POOL" QUEUE_WORKER_VALUE = "1" PYTEST_POOL = "pytest" -PYTEST_WORKER_OPERATIONS = frozenset({"test", "verify_affected"}) +#: Every operation declaring ``pool = "pytest"`` in ``.agentctl/project.toml``, +#: plus the ``test`` operation the pytest slot's own launch document names. A +#: queue runner that does not export ``SINNIXD_QUEUE_POOL`` leaves this the only +#: classifier, and a pytest-pool operation missing here re-queues into the +#: single-slot group its own worker already occupies, which cannot drain. +PYTEST_WORKER_OPERATIONS = frozenset({"test", "verify_affected", "verify_all"}) _CGROUP_PATH = Path("/proc/self/cgroup") _AGENT_CGROUP_SLICES = frozenset({"agent.slice", "sinnixd-pueue-agent.slice"}) _PYTEST_CGROUP_SLICES = frozenset({"sinnixd-pueue-pytest.slice"}) diff --git a/tests/unit/devtools/cgroups.py b/tests/unit/devtools/cgroups.py new file mode 100644 index 0000000000..182e60a4ea --- /dev/null +++ b/tests/unit/devtools/cgroups.py @@ -0,0 +1,53 @@ +"""The cgroup shapes ``devtools.agent_env`` classifies, as the deployment writes them. + +The cgroup is what binds a queue worker's identity to the pool its operation +declared, so a test asserting that decision names the cgroup it means instead of +inheriting whichever one the test session happens to run under. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from devtools import agent_env + +DEPLOYED_AGENT_CGROUP = ( + "0::/user.slice/user-1000.slice/user@1000.service/sinnixd.slice/" + "sinnixd-pueue.slice/sinnixd-pueue-agent.slice/run-p3557538-i192191396.scope\n" +) +DEPLOYED_PYTEST_CGROUP = ( + "0::/user.slice/user-1000.slice/user@1000.service/sinnixd.slice/" + "sinnixd-pueue.slice/sinnixd-pueue-pytest.slice/" + "sinnixd-pueue-pytest-verify_affected-job.scope\n" +) +OUTSIDE_CGROUP = "0::/user.slice/user-1000.slice/user@1000.service/app.slice/shell.scope\n" +WORKFLOW_RUNNER_CGROUP = "0::/sinnixd.slice/sinnixd-work.slice/github-runner-polylogue.service\n" + + +def outside_cgroup() -> str: + return OUTSIDE_CGROUP + + +def deployed_agent_cgroup() -> str: + return DEPLOYED_AGENT_CGROUP + + +def deployed_pytest_cgroup() -> str: + return DEPLOYED_PYTEST_CGROUP + + +def workflow_runner_cgroup() -> str: + return WORKFLOW_RUNNER_CGROUP + + +def stub_cgroup(cgroup: str, *, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Point the ambient cgroup read at ``cgroup``. + + ``devtools.pytest_slot.holds_pytest_slot`` takes no reader, so the file + ``agent_env`` reads is what a test of the whole slot decision controls. + """ + path = tmp_path / "cgroup" + path.write_text(cgroup, encoding="utf-8") + monkeypatch.setattr(agent_env, "_CGROUP_PATH", path) diff --git a/tests/unit/devtools/test_agent_env.py b/tests/unit/devtools/test_agent_env.py index fabe3849f6..c4da9da5b9 100644 --- a/tests/unit/devtools/test_agent_env.py +++ b/tests/unit/devtools/test_agent_env.py @@ -1,35 +1,18 @@ from __future__ import annotations -from devtools import agent_env - -AGENT = {agent_env.AGENT_PRINCIPAL_ENV: agent_env.AGENT_PRINCIPAL} -DEPLOYED_AGENT_CGROUP = ( - "0::/user.slice/user-1000.slice/user@1000.service/sinnixd.slice/" - "sinnixd-pueue.slice/sinnixd-pueue-agent.slice/run-p3557538-i192191396.scope\n" -) -DEPLOYED_PYTEST_CGROUP = ( - "0::/user.slice/user-1000.slice/user@1000.service/sinnixd.slice/" - "sinnixd-pueue.slice/sinnixd-pueue-pytest.slice/" - "sinnixd-pueue-pytest-verify_affected-job.scope\n" -) -OUTSIDE_CGROUP = "0::/user.slice/user-1000.slice/user@1000.service/app.slice/shell.scope\n" -WORKFLOW_RUNNER_CGROUP = "0::/sinnixd.slice/sinnixd-work.slice/github-runner-polylogue.service\n" - - -def outside_cgroup() -> str: - return OUTSIDE_CGROUP - - -def deployed_agent_cgroup() -> str: - return DEPLOYED_AGENT_CGROUP - +from pathlib import Path -def deployed_pytest_cgroup() -> str: - return DEPLOYED_PYTEST_CGROUP +import tomllib +from devtools import agent_env +from tests.unit.devtools.cgroups import ( + deployed_agent_cgroup, + deployed_pytest_cgroup, + outside_cgroup, + workflow_runner_cgroup, +) -def workflow_runner_cgroup() -> str: - return WORKFLOW_RUNNER_CGROUP +AGENT = {agent_env.AGENT_PRINCIPAL_ENV: agent_env.AGENT_PRINCIPAL} def test_outside_agent_jobs_nothing_changes() -> None: @@ -104,3 +87,30 @@ def test_the_workflow_runner_is_not_an_agent_job() -> None: assert agent_env.refuse_verify_tier([], environment, cgroup_reader=workflow_runner_cgroup) is None assert agent_env.refuse_bare_pytest(environment, cgroup_reader=workflow_runner_cgroup) is None assert agent_env.agent_worker_cap(8, environment, cgroup_reader=workflow_runner_cgroup) == 8 + + +def test_every_declared_pytest_pool_operation_classifies_its_own_worker() -> None: + """A pytest-pool worker that queues again waits on the slot it already occupies. + + Anti-vacuity: dropping any pytest-pool operation from + ``PYTEST_WORKER_OPERATIONS`` makes this red, and that deadlock is what a + queue runner not exporting ``SINNIXD_QUEUE_POOL`` would hit. + """ + declarations = tomllib.loads( + (Path(__file__).resolve().parents[3] / ".agentctl" / "project.toml").read_text(encoding="utf-8") + ) + declared = { + name for name, operation in declarations["operations"].items() if operation.get("pool") == agent_env.PYTEST_POOL + } + + assert declared, "the project declares the pytest pool; an empty set would assert nothing" + assert declared <= agent_env.PYTEST_WORKER_OPERATIONS, sorted(declared - agent_env.PYTEST_WORKER_OPERATIONS) + for operation in sorted(declared): + environment = { + **AGENT, + "SINNIXD_JOB_ID": f"{operation}-1", + "SINNIXD_OPERATION": operation, + "SINNIXD_QUEUE_WORKER": "1", + } + + assert agent_env.inside_declared_pytest_worker(environment, cgroup_reader=deployed_pytest_cgroup), operation diff --git a/tests/unit/devtools/test_pytest_slot.py b/tests/unit/devtools/test_pytest_slot.py index a6b63343af..86f4fd9456 100644 --- a/tests/unit/devtools/test_pytest_slot.py +++ b/tests/unit/devtools/test_pytest_slot.py @@ -10,6 +10,11 @@ branch of ``devtools.pytest_slot._runs_pytest_in_place`` makes ``test_a_workflow_job_runs_without_the_local_queue_tools`` red -- the run refuses on the absent queue runner instead of executing. + +Every queueing test here resolves ``pueue`` and ``sinnixd-queue-run`` from +fakes that are the whole PATH, so a green run says nothing about what the +workstation has deployed; the cgroup a slot decision depends on is stubbed for +the same reason. """ from __future__ import annotations @@ -37,9 +42,13 @@ inside_workflow_job, run_pytest, ) +from tests.unit.devtools.cgroups import DEPLOYED_PYTEST_CGROUP, stub_cgroup + +#: ``pueue add`` resolves the runner and records it; the fake ``pueue`` never +#: executes what it was given, so an executable that exits is the whole fake. +FAKE_QUEUE_RUNNER = "raise SystemExit(0)\n" -FAKE_PUEUE = """#!/usr/bin/env python3 -import json, os, sys +FAKE_PUEUE = """import json, os, sys import shutil # The record path is derived from this script's own location: a queued run's @@ -57,6 +66,24 @@ """ +def _install_executable(directory: Path, name: str, source: str) -> Path: + """Install ``source`` as an executable that needs nothing else on PATH. + + The shebang names the interpreter absolutely: these fakes are the entire + PATH of the run under test, so ``/usr/bin/env python3`` would not resolve. + """ + directory.mkdir(parents=True, exist_ok=True) + script = directory / name + script.write_text(f"#!{sys.executable}\n{source}", encoding="utf-8") + script.chmod(script.stat().st_mode | stat.S_IXUSR) + return script + + +def _install_fake_queue_runner(directory: Path) -> Path: + """The runner ``pueue add`` is handed, so queueing never needs the workstation's.""" + return _install_executable(directory, pytest_slot.QUEUE_RUNNER, FAKE_QUEUE_RUNNER) + + def _install_fake_pueue( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -65,18 +92,19 @@ def _install_fake_pueue( result: str = '"Success"', ) -> Path: directory = tmp_path / "fakebin" - directory.mkdir(exist_ok=True) - script = directory / "pueue" - script.write_text( + _install_fake_queue_runner(directory) + script = _install_executable( + directory, + "pueue", FAKE_PUEUE.format( task_id=task_id, result=result, launch_snapshot=str(tmp_path / "queued-launch.json"), ), - encoding="utf-8", ) - script.chmod(script.stat().st_mode | stat.S_IXUSR) - monkeypatch.setenv("PATH", f"{directory}{os.pathsep}{os.environ['PATH']}") + # The fakes are the whole PATH: queueing must resolve its tools from what + # the test installed, never from whatever the workstation has deployed. + monkeypatch.setenv("PATH", str(directory)) return Path(str(script) + ".calls.jsonl") @@ -124,7 +152,7 @@ def test_outside_a_task_the_run_is_queued(tmp_path: Path, monkeypatch: pytest.Mo assert add["argv"][add["argv"].index("--label") + 1] == "polylogue:test:1" assert "--print-task-id" in add["argv"] and "--escape" in add["argv"] assert add["argv"][-2:] == [ - shutil.which(pytest_slot.QUEUE_RUNNER), + str(tmp_path / "fakebin" / pytest_slot.QUEUE_RUNNER), str(tmp_path / ".cache" / "verify" / f"pytest-slot-{os.getpid()}.json"), ] launch = json.loads((tmp_path / "queued-launch.json").read_text(encoding="utf-8")) @@ -180,7 +208,13 @@ def test_lane_job_is_queued_even_with_generic_job_identity(tmp_path: Path, monke def test_declared_pytest_worker_runs_directly(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Queue identity alone is not the slot: the pytest slice is what binds it. + + Anti-vacuity: pointing the stub at any other slice makes this red, because + the worker queues instead of running. + """ record = _install_fake_pueue(tmp_path, monkeypatch) + stub_cgroup(DEPLOYED_PYTEST_CGROUP, tmp_path=tmp_path, monkeypatch=monkeypatch) marker = tmp_path / "pytest-ran" holder = { "SINNIXD_JOB_ID": "job-1", @@ -308,18 +342,27 @@ def test_missing_scoped_queue_runner_refuses_before_queueing(tmp_path: Path, mon def test_an_unreachable_queue_refuses_rather_than_running(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The runner resolves and only the queue is missing, so this names pueue itself. + + Anti-vacuity: dropping the fake runner refuses on the runner instead, which + is ``test_missing_scoped_queue_runner_refuses_before_queueing``; asserting + only the shared advice line would pass either way. + """ marker = tmp_path / "pytest-ran" - monkeypatch.setenv("PATH", str(tmp_path / "empty")) + runner_only = tmp_path / "runner-only" + _install_fake_queue_runner(runner_only) + monkeypatch.setenv("PATH", str(runner_only)) with pytest.raises(PytestSlotUnavailableError) as failure: run_pytest( _marker_command(marker), cwd=str(tmp_path), - env=_environment(PATH=str(tmp_path / "empty")), + env=_environment(PATH=str(runner_only)), root=tmp_path, label="polylogue:test:1", ) + assert "`pueue` is not on PATH" in str(failure.value) assert "systemctl --user start pueued" in str(failure.value) assert not marker.exists() @@ -498,8 +541,7 @@ def test_the_leaked_cloud_basetemp_sentinel_is_declined(tmp_path: Path, monkeypa #: A ``pueue`` whose ``wait`` kills the process waiting on it, the way a #: session or a wrapper being killed leaves a queued task with no waiter. -FAKE_PUEUE_KILLS_ITS_WAITER = """#!/usr/bin/env python3 -import json, os, signal, sys, time +FAKE_PUEUE_KILLS_ITS_WAITER = """import json, os, signal, sys, time with open(sys.argv[0] + ".calls.jsonl", "a", encoding="utf-8") as handle: handle.write(json.dumps({"argv": sys.argv[1:]}) + "\\n") @@ -538,10 +580,8 @@ def test_a_killed_waiter_reaps_the_task_it_queued(tmp_path: Path) -> None: import subprocess directory = tmp_path / "fakebin" - directory.mkdir() - script = directory / "pueue" - script.write_text(FAKE_PUEUE_KILLS_ITS_WAITER, encoding="utf-8") - script.chmod(script.stat().st_mode | stat.S_IXUSR) + _install_fake_queue_runner(directory) + script = _install_executable(directory, "pueue", FAKE_PUEUE_KILLS_ITS_WAITER) record = Path(str(script) + ".calls.jsonl") repo = str(Path(pytest_slot.__file__).resolve().parents[1]) @@ -552,7 +592,7 @@ def test_a_killed_waiter_reaps_the_task_it_queued(tmp_path: Path) -> None: _WAITER.format(repo=repo, cwd=str(tmp_path), root=str(tmp_path)), ], env={ - "PATH": f"{directory}{os.pathsep}{os.environ['PATH']}", + "PATH": str(directory), "HOME": os.environ.get("HOME", "/home/nobody"), }, capture_output=True, From 2be36f35c787d4108930f3e6db369112837ab7d9 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 21:11:09 +0200 Subject: [PATCH 10/47] fix(ci): keep hosted pytest in the host queue --- .github/workflows/verify.yml | 8 ++-- CLAUDE.md | 9 ++-- TESTING.md | 4 -- devtools/pytest_slot.py | 62 +++++-------------------- devtools/verify.py | 1 + tests/unit/devtools/test_agent_env.py | 15 ------ tests/unit/devtools/test_pytest_slot.py | 47 +------------------ 7 files changed, 22 insertions(+), 124 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 143eeb0776..d325a31500 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -2,8 +2,8 @@ name: Verify # The real test evidence for a change. Hosted CI runs only the quick gate, so a # green PR check there does not mean tests ran; this job runs `devtools verify` -# on the workstation runner. One runner executes one job at a time, so pytest -# runs in the job rather than through the workstation's pytest pueue slot. +# on the workstation runner, where pytest goes through the host's single +# `pytest` pueue slot automatically. on: pull_request: merge_group: @@ -56,6 +56,8 @@ jobs: - name: Verify if: steps.pr-state.outputs.skip != 'true' - run: nix develop --command devtools verify + run: | + export PATH="/run/current-system/sw/bin:$PATH" + nix develop --command devtools verify env: POLYLOGUE_FORCE_PLAIN: "1" diff --git a/CLAUDE.md b/CLAUDE.md index 8d25c32850..ee82e2d8f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,11 +148,10 @@ explicit-and-retryable or a typed permanent refusal. a lane with SQLite backup, replacing unusable lane copies. If no seed is available, the run reports a full seed run; `--all` runs every test and updates fingerprints, and `--quick` is the static gates alone. -- Every managed pytest run holds the host's single `pytest` pueue slot. Direct - execution is authorized by the pytest worker's `POLYLOGUE_PYTEST_SLOT=held` - marker or by a GitHub Actions workflow job, which the single self-hosted - runner already serialises. Every other caller queues, waits, reads the - captured log the run prints, and refuses if pueued is unreachable. +- Every managed pytest run holds the host's single `pytest` pueue slot. Only + the pytest worker's `POLYLOGUE_PYTEST_SLOT=held` marker authorizes direct + execution. Every other caller queues, waits, reads the captured log the run + prints, and refuses if pueued is unreachable. - `devtools why` — explain the last run before reading receipts by hand. - `devtools gate ` — one named invariant check (`gate --list`); `verify --quick` is the fast subset. `status`, `render [|all] diff --git a/TESTING.md b/TESTING.md index 0287469af3..f22835afd1 100644 --- a/TESTING.md +++ b/TESTING.md @@ -41,10 +41,6 @@ share one workstation, so `devtools test` and the pytest step of - The pytest-group runner sets `POLYLOGUE_PYTEST_SLOT=held` for the task that owns the slot, so pytest runs in place and streams its output as before. -- A GitHub Actions workflow job (`GITHUB_ACTIONS=true` with a `GITHUB_RUN_ID`) - runs pytest in place too. One self-hosted runner executes one job at a time - under the job plane's slice, so hosted verification is already serialised and - never depends on the workstation's queue tooling being on the runner's PATH. - Every other caller, including a generic AgentCTL job, is queued as `pueue add --group pytest --label polylogue:{test,verify}:`, and the command waits for the task, reports the exit code pueue recorded, and prints the captured log path under diff --git a/devtools/pytest_slot.py b/devtools/pytest_slot.py index d7fd648941..a5a0aee77f 100644 --- a/devtools/pytest_slot.py +++ b/devtools/pytest_slot.py @@ -5,14 +5,11 @@ load control the daemon applies to its own jobs, so concurrent runs contend for the same cores and disk until a long job passes its timeout. -Every managed pytest run started from a session or a lane therefore holds the -host's `pytest` pueue group (one task at a time). A run already inside the -pytest queue task holds the slot already, marked explicitly with -``POLYLOGUE_PYTEST_SLOT=held``. Generic Sinnixd job identity does not imply -pytest-slot ownership: lane jobs queue and wait here. - -A GitHub Actions workflow job is the other execution context that already runs -one at a time, and it runs pytest in place; see :func:`inside_workflow_job`. +Every managed pytest run therefore holds the host's `pytest` pueue group (one +task at a time). A run already inside the pytest queue task holds the slot +already, marked explicitly with ``POLYLOGUE_PYTEST_SLOT=held``. Generic +Sinnixd job identity does not imply pytest-slot ownership: lane jobs queue and +wait here. pueue 4 records the full client environment of ``pueue add`` into a user-only state file, so the adder runs with a reduced environment and the managed pytest @@ -50,7 +47,6 @@ "basetemp_root", "contained_pytest_run", "holds_pytest_slot", - "inside_workflow_job", "main", "remove_temp_tree", "run_pytest", @@ -64,14 +60,6 @@ #: Explicit escape, for the hermetic test of this mechanism. SLOT_ESCAPE_ENV: Final = "POLYLOGUE_PYTEST_SLOT" SLOT_HELD: Final = "held" -#: What the receipt records for a run that executed here as a workflow job. -SLOT_WORKFLOW: Final = "github workflow job" - -#: GitHub Actions sets both in every workflow job. The run id is what -#: distinguishes a job from a shell that merely exported the flag. -WORKFLOW_MARKER_ENV: Final = "GITHUB_ACTIONS" -WORKFLOW_MARKER_VALUE: Final = "true" -WORKFLOW_RUN_ENV: Final = "GITHUB_RUN_ID" #: The host group whose parallelism is one. PYTEST_GROUP: Final = "pytest" @@ -102,8 +90,7 @@ class PytestSlotUnavailableError(RuntimeError): @dataclass(frozen=True) class SlotOutcome: returncode: int - #: What the receipt records: ``pueue task 12``, ``held``, or the - #: context that ran pytest in place. + #: What the receipt records: ``pueue task 12`` or ``held``. slot: str #: Where the queued run's output landed, or None when it streamed. log_path: Path | None = None @@ -194,31 +181,6 @@ def holds_pytest_slot(env: Mapping[str, str]) -> bool: return env.get(SLOT_ESCAPE_ENV) == SLOT_HELD or inside_declared_pytest_worker(env) -def inside_workflow_job(env: Mapping[str, str]) -> bool: - """Whether this process is a GitHub Actions workflow job. - - A workflow job is its own serialisation domain: the repository registers a - single self-hosted runner, which executes one job at a time, and the verify - workflow cancels a superseded run for the same ref. It also runs under the - job plane's slice, so it is inside the load control the pytest slot exists - to give a session subagent. - - Queueing from here would instead make hosted verification depend on the - workstation's private queue installation being on the runner's PATH, and - would leave a queued task to outlive its waiter when GitHub cancels a job. - """ - return env.get(WORKFLOW_MARKER_ENV) == WORKFLOW_MARKER_VALUE and bool(env.get(WORKFLOW_RUN_ENV)) - - -def _runs_pytest_in_place(env: Mapping[str, str]) -> str | None: - """The receipt value for a caller that runs pytest here, or None to queue.""" - if holds_pytest_slot(env): - return SLOT_HELD - if inside_workflow_job(env): - return SLOT_WORKFLOW - return None - - def adder_environment(env: Mapping[str, str]) -> dict[str, str]: """The reduced environment the ``pueue add`` client runs with.""" return {key: env[key] for key in INHERITED_ENVIRONMENT_KEYS if env.get(key)} @@ -494,16 +456,14 @@ def run_pytest( """Run a managed pytest command, acquiring the host's pytest slot first. When the pytest-group runner sets ``POLYLOGUE_PYTEST_SLOT=held``, the slot - is already held and the command runs here, streaming as before; a GitHub - Actions workflow job runs here for the reason :func:`inside_workflow_job` - gives. Every other caller is queued in the host's single-slot ``pytest`` - group and its output is captured. + is already held and the command runs here, streaming as before. Every other + caller is queued in the host's single-slot ``pytest`` group and its output + is captured. """ argv, contained, scratch = contained_pytest_run(command, env=env, root=root) - in_place = _runs_pytest_in_place(env) - if in_place is not None: + if holds_pytest_slot(env): completed = subprocess.run(argv, cwd=cwd, env=contained, stdout=stdout, stderr=stdout) - outcome = SlotOutcome(returncode=completed.returncode, slot=in_place) + outcome = SlotOutcome(returncode=completed.returncode, slot=SLOT_HELD) else: outcome = _queue(argv, cwd=cwd, env=contained, root=root, label=label) # A failed run keeps its scratch: that is when the leftovers are worth diff --git a/devtools/verify.py b/devtools/verify.py index a38cd10fdc..8fd2a242c1 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -89,6 +89,7 @@ "tests/unit/devtools/test_deployment_browser_smoke_service.py::test_sinnixd_parser_accepts_the_unleased_shared_chrome_operation", "tests/unit/devtools/test_dev_loop_service.py::test_declared_operation_has_a_json_contract_and_no_retired_keys", "tests/unit/devtools/test_seeded_archive_cache_gc.py::test_declared_agentctl_operation_is_bounded_and_previewable", + "tests/unit/devtools/test_agent_env.py::test_every_declared_pytest_pool_operation_classifies_its_own_worker", "tests/unit/devtools/test_verify.py::test_verify_quick_descriptor_accepts_the_declared_json_projection", ) _UNMEASURED_WORKLOAD_DIMENSIONS = ( diff --git a/tests/unit/devtools/test_agent_env.py b/tests/unit/devtools/test_agent_env.py index c4da9da5b9..8cc10f4f66 100644 --- a/tests/unit/devtools/test_agent_env.py +++ b/tests/unit/devtools/test_agent_env.py @@ -9,7 +9,6 @@ deployed_agent_cgroup, deployed_pytest_cgroup, outside_cgroup, - workflow_runner_cgroup, ) AGENT = {agent_env.AGENT_PRINCIPAL_ENV: agent_env.AGENT_PRINCIPAL} @@ -75,20 +74,6 @@ def test_non_pytest_queue_worker_remains_agent_bound() -> None: assert agent_env.refuse_bare_pytest(environment, cgroup_reader=deployed_agent_cgroup) is not None -def test_the_workflow_runner_is_not_an_agent_job() -> None: - """Hosted verification runs the test tier; widening the agent slices would refuse it. - - Anti-vacuity: adding the runner's ``sinnixd-work.slice`` to the agent slices - makes every assertion here red. - """ - environment = {"GITHUB_ACTIONS": "true", "GITHUB_RUN_ID": "33895265963", "CI": "true"} - - assert not agent_env.inside_agent_job(environment, cgroup_reader=workflow_runner_cgroup) - assert agent_env.refuse_verify_tier([], environment, cgroup_reader=workflow_runner_cgroup) is None - assert agent_env.refuse_bare_pytest(environment, cgroup_reader=workflow_runner_cgroup) is None - assert agent_env.agent_worker_cap(8, environment, cgroup_reader=workflow_runner_cgroup) == 8 - - def test_every_declared_pytest_pool_operation_classifies_its_own_worker() -> None: """A pytest-pool worker that queues again waits on the slot it already occupies. diff --git a/tests/unit/devtools/test_pytest_slot.py b/tests/unit/devtools/test_pytest_slot.py index 86f4fd9456..abfc88a06b 100644 --- a/tests/unit/devtools/test_pytest_slot.py +++ b/tests/unit/devtools/test_pytest_slot.py @@ -6,10 +6,7 @@ ``test_the_adder_environment_carries_only_the_allowed_keys`` red. Dropping either half of the temporary-directory containment (the ``--basetemp`` argument or the exported TMPDIR) makes -``test_a_queued_run_contains_its_temporary_trees`` red. Removing the workflow -branch of ``devtools.pytest_slot._runs_pytest_in_place`` makes -``test_a_workflow_job_runs_without_the_local_queue_tools`` red -- the run -refuses on the absent queue runner instead of executing. +``test_a_queued_run_contains_its_temporary_trees`` red. Every queueing test here resolves ``pueue`` and ``sinnixd-queue-run`` from fakes that are the whole PATH, so a green run says nothing about what the @@ -35,11 +32,9 @@ from devtools import cloud_sentinels, pytest_slot from devtools.pytest_slot import ( BASETEMP_ROOT_ENV, - SLOT_WORKFLOW, PytestSlotUnavailableError, basetemp_root, holds_pytest_slot, - inside_workflow_job, run_pytest, ) from tests.unit.devtools.cgroups import DEPLOYED_PYTEST_CGROUP, stub_cgroup @@ -258,46 +253,6 @@ def test_explicit_slot_holder_runs_directly(tmp_path: Path, monkeypatch: pytest. assert _calls(record) == [], "a slot holder must not talk to pueue" -def test_a_workflow_job_runs_without_the_local_queue_tools(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Hosted verification must not depend on the workstation's queue installation.""" - empty = tmp_path / "empty" - empty.mkdir() - monkeypatch.setenv("PATH", str(empty)) - marker = tmp_path / "pytest-ran" - workflow = _environment(PATH=str(empty), GITHUB_ACTIONS="true", GITHUB_RUN_ID="33895265963") - - assert not holds_pytest_slot(workflow), "a workflow job does not own the host slot" - assert inside_workflow_job(workflow) - outcome = run_pytest( - _marker_command(marker), - cwd=str(tmp_path), - env=workflow, - root=tmp_path, - label="polylogue:verify:1", - ) - - assert marker.exists(), "the workflow job did not run pytest" - assert (outcome.returncode, outcome.slot) == (0, SLOT_WORKFLOW) - - -def test_a_workflow_flag_without_a_run_identity_still_queues(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """The exported flag alone is not a workflow job.""" - record = _install_fake_pueue(tmp_path, monkeypatch) - marker = tmp_path / "pytest-ran" - - outcome = run_pytest( - _marker_command(marker), - cwd=str(tmp_path), - env=_environment(GITHUB_ACTIONS="true"), - root=tmp_path, - label="polylogue:verify:1", - ) - - assert not marker.exists() - assert outcome.slot == "pueue task 7" - assert _calls(record)[0]["argv"][0] == "add" - - def test_an_agent_pool_worker_queues_its_focused_run_in_the_pytest_pool( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 9e56d040407a7ac014a8742836f96d635fa8db57 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 01:00:15 +0200 Subject: [PATCH 11/47] test(devtools): pin the corpus operation's checkout and schedule verify_all's absence from per-lane publication rests on it being default-checkout-only and scheduled; assert both alongside the publication list so restoring it to a lane fails here. Co-Authored-By: Claude Opus 5 --- tests/unit/devtools/test_verify.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 9fc942a2d9..1876fa1014 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -306,6 +306,8 @@ def test_verify_quick_descriptor_accepts_the_declared_json_projection() -> None: assert affected["pool"] == "pytest" assert affected["result"] == "pytest" assert complete["exec"] == ["env", "POLYLOGUE_PYTEST_WORKERS=2", "devtools", "verify", "--all"] + assert complete["checkout"] == "default" + assert complete["schedule"] == "*-*-* 03:17:00" assert complete["pool"] == "pytest" assert projection["kind"] == "polylogue.verification-result" assert projection["operation"] == "verify_quick" From 521ae22864196081ebf5b66511175d2ffeeb2af0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 01:00:50 +0200 Subject: [PATCH 12/47] fix(devtools): reap the pytest slot through agentctl pueue kills the queue runner with SIGKILL, so the workload it started survives in the transient systemd scope pueue cannot see: a reaped waiter left pytest running against the checkout while the slot showed free. `agentctl job cancel` stops the task and then empties that scope. The launch file carries the resolved environment and belongs to whoever ends the run, so it is deleted only when the cancellation succeeded -- a task still on the queue reads it when it starts. Co-Authored-By: Claude Opus 5 --- devtools/pytest_slot.py | 46 +++++++++++++---- tests/unit/devtools/test_pytest_slot.py | 66 +++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/devtools/pytest_slot.py b/devtools/pytest_slot.py index a5a0aee77f..f372470569 100644 --- a/devtools/pytest_slot.py +++ b/devtools/pytest_slot.py @@ -209,6 +209,28 @@ def _pueue(arguments: Sequence[str], *, env: Mapping[str, str]) -> subprocess.Co ) from exc +def _cancel_task(task_id: str, *, env: Mapping[str, str]) -> None: + """Cancel through the owner that also empties the task's systemd scope.""" + executable = shutil.which("agentctl", path=env.get("PATH") or os.defpath) + if executable is None: + raise PytestSlotUnavailableError(REFUSAL.format(reason="`agentctl` is not on PATH")) + try: + completed = subprocess.run( + [executable, "job", "cancel", task_id], + env=dict(env), + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + raise PytestSlotUnavailableError( + REFUSAL.format(reason=f"`agentctl job cancel` could not start: {exc}") + ) from exc + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise PytestSlotUnavailableError(REFUSAL.format(reason=f"`agentctl job cancel` failed: {detail}")) + + def _task_result(status_json: str, task_id: str) -> int: try: document = json.loads(status_json) @@ -231,20 +253,26 @@ def _task_result(status_json: str, task_id: str) -> int: def _reap_task(task_id: str, *, env: Mapping[str, str], launch_path: Path | None = None) -> None: - """End a task this process owns and drop it from the queue. + """End a task this process owns and empty its execution scope. Best effort by construction: the reason we are here is that the waiter is being killed, so a failing reap must not replace the original cause of - death with its own error. ``kill`` first because ``remove`` refuses a - running task; ``remove`` after because a killed task still occupies the - listing. The launch file carries a resolved environment and must not - outlive the run either way. + death with its own error. AgentCTL owns cancellation because pueue kills + the queue runner with SIGKILL, which leaves the workload alive in the + transient systemd scope that only AgentCTL names and stops. + + The launch file belongs to whoever ends the run, so deleting it is part of + a cancellation that succeeded: a task still on the queue reads it when it + starts. """ - for verb in ("kill", "remove"): - with contextlib.suppress(PytestSlotUnavailableError): - _pueue([verb, task_id], env=env) - if launch_path is not None: + cancelled = False + try: + _cancel_task(task_id, env=env) + cancelled = True + except PytestSlotUnavailableError: + pass + if cancelled and launch_path is not None: with contextlib.suppress(OSError): launch_path.unlink(missing_ok=True) diff --git a/tests/unit/devtools/test_pytest_slot.py b/tests/unit/devtools/test_pytest_slot.py index abfc88a06b..f7413dc03e 100644 --- a/tests/unit/devtools/test_pytest_slot.py +++ b/tests/unit/devtools/test_pytest_slot.py @@ -510,6 +510,23 @@ def test_the_leaked_cloud_basetemp_sentinel_is_declined(tmp_path: Path, monkeypa sys.exit(0) """ +#: The cancellation AgentCTL owns, recorded the way the ``pueue`` fake records. +FAKE_AGENTCTL = """import json, sys + +with open(sys.argv[0] + ".calls.jsonl", "a", encoding="utf-8") as handle: + handle.write(json.dumps({"argv": sys.argv[1:]}) + "\\n") +sys.exit(0) +""" + +#: An AgentCTL that refuses, the way a job id pueue no longer holds refuses. +FAKE_AGENTCTL_REFUSES = """import json, sys + +with open(sys.argv[0] + ".calls.jsonl", "a", encoding="utf-8") as handle: + handle.write(json.dumps({"argv": sys.argv[1:]}) + "\\n") +sys.stderr.write("agentctl: no such job\\n") +sys.exit(1) +""" + _WAITER = """ import os, sys sys.path.insert(0, {repo!r}) @@ -528,16 +545,20 @@ def test_the_leaked_cloud_basetemp_sentinel_is_declined(tmp_path: Path, monkeypa def test_a_killed_waiter_reaps_the_task_it_queued(tmp_path: Path) -> None: """A task outlives its waiter, and the slot's parallelism is one. - Anti-vacuity: dropping the ``_reaping`` context leaves the recorded calls - at ``add``/``wait`` -- the task stays queued with nothing left to wait on - it, which is exactly the starvation this reap exists to prevent. + Anti-vacuity: dropping the ``_reaping`` context records no cancellation at + all -- the task stays queued with nothing left to wait on it, which is + exactly the starvation this reap exists to prevent. Reaping through + ``pueue kill`` instead leaves the workload running in its scope, and shows + up here as a ``kill`` among the pueue verbs. """ import subprocess directory = tmp_path / "fakebin" _install_fake_queue_runner(directory) script = _install_executable(directory, "pueue", FAKE_PUEUE_KILLS_ITS_WAITER) + agentctl = _install_executable(directory, "agentctl", FAKE_AGENTCTL) record = Path(str(script) + ".calls.jsonl") + cancellation_record = Path(str(agentctl) + ".calls.jsonl") repo = str(Path(pytest_slot.__file__).resolve().parents[1]) completed = subprocess.run( @@ -557,8 +578,45 @@ def test_a_killed_waiter_reaps_the_task_it_queued(tmp_path: Path) -> None: assert completed.returncode == -int(signal.SIGTERM), completed.stderr verbs = [call["argv"][0] for call in _calls(record)] - assert verbs == ["add", "wait", "kill", "remove"], verbs + assert verbs == ["add", "wait"], verbs + assert _calls(cancellation_record) == [{"argv": ["job", "cancel", "11"]}] leftover = list((tmp_path / "verify").glob("pytest-slot-*.json")) + list( (tmp_path / ".cache" / "verify").glob("pytest-slot-*.json") ) assert leftover == [], "the launch file carries a resolved environment and must not survive the reap" + + +def test_a_refused_cancellation_leaves_the_launch_file_for_the_task(tmp_path: Path) -> None: + """A task AgentCTL would not stop still reads its launch file when it starts. + + Anti-vacuity: unlinking the launch file regardless of the cancellation's + outcome empties the glob below, and the surviving task then starts with no + resolved environment to read. + """ + import subprocess + + directory = tmp_path / "fakebin" + _install_fake_queue_runner(directory) + _install_executable(directory, "pueue", FAKE_PUEUE_KILLS_ITS_WAITER) + agentctl = _install_executable(directory, "agentctl", FAKE_AGENTCTL_REFUSES) + repo = str(Path(pytest_slot.__file__).resolve().parents[1]) + + completed = subprocess.run( + [ + sys.executable, + "-c", + _WAITER.format(repo=repo, cwd=str(tmp_path), root=str(tmp_path)), + ], + env={ + "PATH": str(directory), + "HOME": os.environ.get("HOME", "/home/nobody"), + }, + capture_output=True, + text=True, + timeout=60, + ) + + assert completed.returncode == -int(signal.SIGTERM), completed.stderr + assert _calls(Path(str(agentctl) + ".calls.jsonl")) == [{"argv": ["job", "cancel", "11"]}] + surviving = list((tmp_path / ".cache" / "verify").glob("pytest-slot-*.json")) + assert len(surviving) == 1, surviving From b690f13b3a710944fb22da80846b82467da8fe66 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 01:01:02 +0200 Subject: [PATCH 13/47] chore(lane): stop tracking lane publication text #4531 untracked `.lane/`; #4649 re-added `.lane/title` with it, so every lane that writes its own subject dirties a tracked file and trips the worktree identity check. `.gitignore` already covers the directory. Co-Authored-By: Claude Opus 5 --- .lane/title | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .lane/title diff --git a/.lane/title b/.lane/title deleted file mode 100644 index 132246bb77..0000000000 --- a/.lane/title +++ /dev/null @@ -1 +0,0 @@ -refactor: dissolve private session insight lifecycle From ba262cee4926a82803e80ab7421aea085dad82bd Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 03:00:24 +0200 Subject: [PATCH 14/47] docs: describe required affected PR verification --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ee82e2d8f0..a3544fb1dc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -169,8 +169,8 @@ condition — what mutation or bypass would make it red. Fixtures are generated and deterministic (`tests/infra/`: SessionBuilder, seeded archives, pathology composer, corpus programs); timestamp-sensitive tests use `frozen_clock` (an autouse guard rejects wall-clock reads). Keep -ambient machine data out of tests. Per-PR CI runs only the quick gate — a -green PR check does not mean tests ran; verify locally. +ambient machine data out of tests. The required per-PR `verify` check runs +affected pytest through the host slot; the quick gate remains a separate check. Change cross-checks: parser/detection → origin specs + real fixtures + replay parity; storage/schema → fresh DDL + declared lifecycle + readers/writers + From de5585b73a2de2b3526a3a02ecfa53189b3bab1b Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 11:21:36 +0200 Subject: [PATCH 15/47] refactor: route CLI status through operation kernel --- polylogue/cli/commands/status.py | 200 +++++++++++++----------- polylogue/cli/operation_kernel.py | 38 ++++- tests/unit/cli/test_operation_kernel.py | 33 ++++ 3 files changed, 180 insertions(+), 91 deletions(-) diff --git a/polylogue/cli/commands/status.py b/polylogue/cli/commands/status.py index 11ac273d66..0ee48253ee 100644 --- a/polylogue/cli/commands/status.py +++ b/polylogue/cli/commands/status.py @@ -233,6 +233,62 @@ def _fetch_uds_operation(config: Any, operation: str) -> dict[str, Any] | None: return result.value if isinstance(result.value, dict) else None +def _status_operation_result( + env: AppEnv, *, daemon_url: str | None = None, include_archive_readiness: bool = False +) -> Any: + """Execute status once through the operation kernel for every authority.""" + from polylogue.cli.daemon_client import DaemonClient + from polylogue.cli.operation_kernel import OperationKernel, OperationRequest + from polylogue.cli.shared.helpers import load_effective_config + from polylogue.daemon.api_auth import resolve_api_auth_token + from polylogue.daemon.socket_path import daemon_socket_path + from polylogue.version import POLYLOGUE_VERSION + + config = load_effective_config(env) if daemon_url in (None, _BUILTIN_DAEMON_URL) else None + client = ( + DaemonClient( + daemon_socket_path(config.archive_root), + timeout_s=_FULL_TIMEOUT_S, + auth_token=resolve_api_auth_token( + getattr(config, "api_auth_token", None), + allow_no_auth=getattr(config, "api_allow_no_auth", False), + ), + ) + if config is not None + else None + ) + request = OperationRequest("status", {"include_archive_readiness": include_archive_readiness}) + + def daemon_call(lowered: Any) -> Any: + if daemon_url is None or daemon_url == _BUILTIN_DAEMON_URL: + assert config is not None and client is not None + return client.operation( + lowered.operation, + dict(lowered.payload), + archive_root=str(config.archive_root), + daemon_version=POLYLOGUE_VERSION, + ) + for candidate in _candidate_daemon_urls(daemon_url): + try: + request = Request(f"{candidate}/api/status", headers={"Accept": "application/json"}, method="GET") + with urlopen(request, timeout=_FULL_TIMEOUT_S) as response: + value = json.loads(response.read()) + return {"operation": lowered.operation, "result": value, "authority": {"mode": "daemon"}} + except (OSError, TimeoutError, ValueError): + continue + return None + + return OperationKernel( + daemon_call, + lambda _lowered: _show_direct_json( + env, + full=True, + include_archive_readiness=include_archive_readiness, + emit=False, + ), + ).execute(request) + + def _candidate_daemon_urls(primary_url: str) -> tuple[str, ...]: """Return daemon URLs worth probing, with explicit config first. @@ -935,66 +991,44 @@ def status_command( route="cli.status", verb="full" if full_payload else "compact", ) as obs: - if daemon_url == _BUILTIN_DAEMON_URL: - try: - from polylogue.cli.shared.helpers import load_effective_config - - result = _fetch_uds_operation(load_effective_config(env), "status") - except Exception: - result = None - if result is not None: - obs.attributes["daemon_reachable"] = True - obs.daemon_path = "daemon" - if output_format == "json": - status_ok = _show_status_json(env, result, full=full_payload) - else: - status_ok = _show_daemon_status(env, result) - if not status_ok: - raise click.exceptions.Exit(1) - return - candidate_urls = _candidate_daemon_urls(daemon_url) - for candidate_url in candidate_urls: - try: - req = Request( - f"{candidate_url}/api/status", - headers={"Accept": "application/json"}, - method="GET", - ) - with urlopen(req, timeout=_FULL_TIMEOUT_S) as resp: - result = json.loads(resp.read()) - except (OSError, ValueError): - # ValueError covers malformed URLs (urllib raises before any I/O). - continue - obs.attributes["daemon_reachable"] = True - obs.daemon_path = "daemon" + try: + operation_result = _status_operation_result( + env, + daemon_url=daemon_url, + include_archive_readiness=exact_archive_readiness, + ) + except Exception: + operation_result = None + if operation_result is None: + obs.attributes["daemon_reachable"] = False + obs.daemon_path = "direct" if output_format == "json": - status_ok = _show_status_json(env, result, full=full_payload) - else: - status_ok = _show_daemon_status(env, result) - if not status_ok: - raise click.exceptions.Exit(1) - return - - obs.attributes["daemon_reachable"] = False - obs.daemon_path = "direct" - if output_format == "json": - if any(_daemon_live(url, timeout=_FAST_TIMEOUT_S) for url in candidate_urls): - obs.status = "degraded" - _show_daemon_status_unavailable_json(env) - raise click.exceptions.Exit(1) - else: status_ok = _show_direct_json(env, full=full_payload, include_archive_readiness=exact_archive_readiness) - if not status_ok: - raise click.exceptions.Exit(1) - else: - if any(_daemon_live(url, timeout=_FAST_TIMEOUT_S) for url in candidate_urls): - obs.status = "degraded" - _show_daemon_status_unavailable(env) - raise click.exceptions.Exit(1) else: status_ok = _show_direct_status(env, include_archive_readiness=exact_archive_readiness) - if not status_ok: - raise click.exceptions.Exit(1) + else: + mode = operation_result.authority.get("mode") + obs.attributes["daemon_reachable"] = mode == "daemon" + obs.daemon_path = str(mode) + status = operation_result.value + if mode == "direct": + status_ok = ( + _show_direct_json( + env, + full=full_payload, + include_archive_readiness=exact_archive_readiness, + ) + if output_format == "json" + else _show_direct_status(env, include_archive_readiness=exact_archive_readiness) + ) + else: + status_ok = ( + _show_status_json(env, status, full=full_payload) + if output_format == "json" + else _show_daemon_status(env, status) + ) + if not status_ok: + raise click.exceptions.Exit(1) return @@ -1004,36 +1038,18 @@ def show_fast_status(env: AppEnv, *, daemon_url: str | None = None) -> None: Called from ``polylogue`` with no args. Uses a short HTTP timeout and bounded SQLite queries to stay under 2 seconds. """ - resolved_url = daemon_url if daemon_url is not None else _default_daemon_url() - if resolved_url == _BUILTIN_DAEMON_URL: - try: - from polylogue.cli.shared.helpers import load_effective_config + del daemon_url + from polylogue.cli.operation_kernel import OperationKernelError - result = _fetch_uds_operation(load_effective_config(env), "status") - except Exception: - result = None - if result is not None: - _show_daemon_status(env, result, compact=True) - return - candidate_urls = _candidate_daemon_urls(resolved_url) - for candidate_url in candidate_urls: - try: - req = Request( - f"{candidate_url}/api/status", - headers={"Accept": "application/json"}, - method="GET", - ) - with urlopen(req, timeout=_FAST_TIMEOUT_S) as resp: - result = json.loads(resp.read()) - except (OSError, ValueError): - continue - _show_daemon_status(env, result, compact=True) + try: + result = _status_operation_result(env) + except OperationKernelError: + _show_direct_status(env, compact=True) return - - if any(_daemon_live(url, timeout=_FAST_TIMEOUT_S) for url in candidate_urls): - _show_daemon_status_unavailable(env, compact=True) - else: + if result.authority.get("mode") == "direct": _show_direct_status(env, compact=True) + else: + _show_daemon_status(env, result.value, compact=True) def _show_daemon_status(env: AppEnv, status: dict[str, Any], *, compact: bool = False) -> bool: @@ -1163,8 +1179,13 @@ def _show_daemon_status(env: AppEnv, status: dict[str, Any], *, compact: bool = def _show_status_json(env: AppEnv, status: dict[str, Any], *, full: bool = False) -> bool: """Machine-readable JSON status output.""" - normalized = normalize_raw_frontier_status_payload(status, require_fresh_snapshot=True) - payload = normalized if full else _compact_status_payload(normalized, source="daemon") + source = "direct" if status.get("daemon_liveness") is False else "daemon" + normalized = normalize_raw_frontier_status_payload( + status, + snapshot_state="live" if source == "direct" else None, + require_fresh_snapshot=source == "daemon", + ) + payload = normalized if full else _compact_status_payload(normalized, source=source) env.ui.console.print(json.dumps(payload, indent=2, default=str)) return _status_ok(normalized, require_fresh_snapshot=True) @@ -1505,7 +1526,8 @@ def _show_direct_json( *, full: bool = False, include_archive_readiness: bool = False, -) -> bool: + emit: bool = True, +) -> bool | dict[str, Any]: """Machine-readable JSON fallback when daemon is not running.""" from polylogue.cli.commands.init import starter_config_path from polylogue.cli.commands.status_diagnostics import ( @@ -1616,8 +1638,10 @@ def _show_direct_json( if full or include_archive_readiness else _compact_status_payload(normalized_payload, source="direct") ) - env.ui.console.print(json.dumps(output, indent=2, default=str)) - return _status_ok(normalized_payload) + if emit: + env.ui.console.print(json.dumps(output, indent=2, default=str)) + return _status_ok(normalized_payload) + return cast(dict[str, Any], normalized_payload) def _component_computation_failure(component: str, exc: Exception, *, scope: str = "archive") -> dict[str, Any]: diff --git a/polylogue/cli/operation_kernel.py b/polylogue/cli/operation_kernel.py index 969a195d50..d21602904e 100644 --- a/polylogue/cli/operation_kernel.py +++ b/polylogue/cli/operation_kernel.py @@ -9,11 +9,13 @@ from __future__ import annotations +import json from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Any from polylogue.operations.daemon_protocol import ( + MAX_OPERATION_RESULT_BYTES, DaemonAuthority, DaemonOperationSpec, daemon_operation_spec, @@ -41,6 +43,14 @@ def __init__(self, code: str, detail: object = None) -> None: super().__init__(f"{code}: {detail}" if detail else code) +def _result_size(value: object) -> int: + """Return the bounded wire size of a JSON-compatible operation result.""" + try: + return len(json.dumps(value, separators=(",", ":"), default=str).encode()) + except (TypeError, ValueError, OverflowError) as exc: + raise OperationEnvelopeError("operation result is not JSON serializable") from exc + + @dataclass(frozen=True, slots=True) class OperationRequest: """A lowered operation request; no surface-specific query vocabulary.""" @@ -85,7 +95,17 @@ def __init__(self, daemon_call: OperationCall, direct_call: DirectCall | None = def execute(self, request: OperationRequest) -> OperationResult: spec = request.spec - envelope = self._daemon_call(request) + try: + envelope = self._daemon_call(request) + except (TimeoutError, ConnectionError, OSError): + envelope = None + except Exception as exc: + # The stdlib daemon client uses a protocol error for bounded-result + # violations. Keep that distinction visible to callers while + # preserving direct fallback for ordinary daemon absence. + if type(exc).__name__ == "DaemonOperationProtocolError" and "size" in str(exc): + raise OperationFailedError("result_too_large", str(exc)) from exc + raise OperationFailedError("daemon_transport_error", str(exc)) from exc if envelope is not None: if envelope.get("operation") not in (None, request.operation): raise OperationEnvelopeError("daemon returned a different operation") @@ -101,19 +121,31 @@ def execute(self, request: OperationRequest) -> OperationResult: if "result" not in envelope: raise OperationEnvelopeError("daemon response omitted the operation result") value = envelope.get("result") + if _result_size(value) > MAX_OPERATION_RESULT_BYTES: + raise OperationFailedError("result_too_large", "daemon operation result exceeds the bounded size") + generation = envelope.get("generation") + if isinstance(generation, Mapping) and generation.get("state") in {"stale", "mismatch"}: + raise OperationFailedError("stale_generation", generation.get("reason")) authority = envelope.get("authority") + if not isinstance(authority, Mapping): + authority = {"mode": "daemon", "class": spec.authority.value} + else: + authority = {"mode": "daemon", "class": spec.authority.value, **authority} return OperationResult( request.operation, value, - authority if isinstance(authority, Mapping) else {"mode": "daemon", "class": spec.authority.value}, + authority, envelope, ) if not spec.direct_allowed or spec.authority is not DaemonAuthority.READ or self._direct_call is None: raise OperationUnavailableError(f"daemon is unavailable for operation: {request.operation}") + value = self._direct_call(request) + if _result_size(value) > MAX_OPERATION_RESULT_BYTES: + raise OperationFailedError("result_too_large", "direct operation result exceeds the bounded size") return OperationResult( request.operation, - self._direct_call(request), + value, {"mode": "direct", "class": spec.authority.value, "fallback": spec.fallback.value}, ) diff --git a/tests/unit/cli/test_operation_kernel.py b/tests/unit/cli/test_operation_kernel.py index 40a7cacd62..b1b70131f4 100644 --- a/tests/unit/cli/test_operation_kernel.py +++ b/tests/unit/cli/test_operation_kernel.py @@ -81,3 +81,36 @@ def test_non_read_operation_cannot_use_direct_fallback() -> None: ) finally: daemon_protocol.DAEMON_OPERATION_SPECS = original + + +@pytest.mark.parametrize( + ("envelope", "code"), + [ + ({"outcome": "cancelled", "result": None}, "cancelled"), + ({"outcome": "timeout", "result": None}, "timeout"), + ({"generation": {"state": "stale"}, "result": {}}, "stale_generation"), + ], +) +def test_terminal_and_stale_daemon_states_are_typed(envelope: dict[str, object], code: str) -> None: + with pytest.raises(OperationFailedError) as exc_info: + OperationKernel(lambda _request: envelope).execute(OperationRequest("cli.query", {})) + assert exc_info.value.code == code + + +def test_oversized_result_is_rejected_before_rendering() -> None: + from polylogue.operations.daemon_protocol import MAX_OPERATION_RESULT_BYTES + + with pytest.raises(OperationFailedError) as exc_info: + OperationKernel(lambda _request: {"result": "x" * (MAX_OPERATION_RESULT_BYTES + 1)}).execute( + OperationRequest("cli.query", {}) + ) + assert exc_info.value.code == "result_too_large" + + +def test_timeout_falls_back_to_direct_read() -> None: + result = OperationKernel( + lambda _request: (_ for _ in ()).throw(TimeoutError("deadline")), + lambda _request: {"items": []}, + ).execute(OperationRequest("cli.query", {})) + assert result.value == {"items": []} + assert result.authority["mode"] == "direct" From 47c6d037605e7a645dc6516bc51a674e4159aa03 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 20:37:11 +0200 Subject: [PATCH 16/47] fix: Invalidate derived session products on late-parent resolution `_reextract_prefix_tail_db` deleted from `insight_materialization`, which #4649 retired from the index DDL, so every deferred-tail resolution raised `no such table`. Delete the child's `session_profiles`, `session_latency_profiles`, `session_work_events` and `session_phases` instead: their staleness predicate compares the session's sort key, updated-at and content hash, none of which re-extraction moves, so a profile materialized over the whole child would report fresh indefinitely. The same retirement left four `SessionInsightCountDescriptor`s whose `count_key`s are no longer snapshot fields and whose `table_key` names the retired table, making every `session_insight_status_sync`/`_async` call raise `KeyError`. Remove them and their SQL. Co-Authored-By: Claude Opus 5 --- docs/schema.md | 4 +- polylogue/storage/derived/session/status.py | 44 -------- .../storage/sqlite/archive_tiers/write.py | 9 +- .../storage/test_lineage_normalization.py | 100 +++++++++++++++++- ...test_session_insight_status_descriptors.py | 41 +++++++ 5 files changed, 149 insertions(+), 49 deletions(-) diff --git a/docs/schema.md b/docs/schema.md index c337ae8cd5..cfa437b283 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -93,8 +93,8 @@ compact indexed `action_pairs` and `delegation_facts` relations, query-time `session_model_usage`, the auto-tag side of `session_tags`, and the insight read models (`session_profiles`, `session_work_events`, `session_phases`, -`session_latency_profiles`) plus -`insight_materialization` for cache invalidation. +`session_latency_profiles`), whose freshness the converger derives from the +owning session's sort key and content hash. ### `embeddings.db` — vectors (rebuildable, expensive) diff --git a/polylogue/storage/derived/session/status.py b/polylogue/storage/derived/session/status.py index d480476c08..9bd3f25c90 100644 --- a/polylogue/storage/derived/session/status.py +++ b/polylogue/storage/derived/session/status.py @@ -219,24 +219,6 @@ async def count_async( LEFT JOIN sessions c ON c.session_id = slp.session_id WHERE c.session_id IS NULL """ -MISSING_INSIGHT_MATERIALIZATION_COUNT_SQL = """ - SELECT COUNT(*) - FROM sessions c - WHERE NOT EXISTS ( - SELECT 1 - FROM insight_materialization im - WHERE im.session_id = c.session_id - AND im.insight_type = ? - AND im.materializer_version = ? - AND ABS(COALESCE(im.source_sort_key_ms, 0) - COALESCE(c.sort_key_ms, 0)) = 0 - ) -""" -MISSING_THREAD_MATERIALIZATION_COUNT_SQL = """ - SELECT COUNT(*) - FROM sessions c - LEFT JOIN session_profiles sp ON sp.session_id = c.session_id - WHERE sp.session_id IS NULL -""" EXPECTED_WORK_EVENT_COUNT_SQL = "SELECT COALESCE(SUM(work_event_count), 0) FROM session_profiles" EXPECTED_PHASE_COUNT_SQL = "SELECT COALESCE(SUM(phase_count), 0) FROM session_profiles" ORPHAN_SESSION_WORK_EVENT_COUNT_SQL = """ @@ -514,32 +496,6 @@ async def _stale_session_profile_count_sql_async(conn: aiosqlite.Connection) -> sql=ORPHAN_SESSION_PHASE_COUNT_SQL, requires_freshness=True, ), - SessionInsightCountDescriptor( - count_key="missing_run_materialization_count", - table_key="insight_materialization", - sql=MISSING_INSIGHT_MATERIALIZATION_COUNT_SQL, - params=("runs", SESSION_INSIGHT_MATERIALIZER_VERSION), - fallback_count_key="total_sessions", - ), - SessionInsightCountDescriptor( - count_key="missing_observed_event_materialization_count", - table_key="insight_materialization", - sql=MISSING_INSIGHT_MATERIALIZATION_COUNT_SQL, - params=("observed_events", SESSION_INSIGHT_MATERIALIZER_VERSION), - fallback_count_key="total_sessions", - ), - SessionInsightCountDescriptor( - count_key="missing_context_snapshot_materialization_count", - table_key="insight_materialization", - sql=MISSING_INSIGHT_MATERIALIZATION_COUNT_SQL, - params=("context_snapshots", SESSION_INSIGHT_MATERIALIZER_VERSION), - fallback_count_key="total_sessions", - ), - SessionInsightCountDescriptor( - count_key="missing_thread_materialization_count", - sql=MISSING_THREAD_MATERIALIZATION_COUNT_SQL, - fallback_count_key="total_sessions", - ), SessionInsightCountDescriptor( count_key="stale_thread_count", sql=STALE_THREAD_COUNT_SQL, diff --git a/polylogue/storage/sqlite/archive_tiers/write.py b/polylogue/storage/sqlite/archive_tiers/write.py index 074762554a..4fd60ad874 100644 --- a/polylogue/storage/sqlite/archive_tiers/write.py +++ b/polylogue/storage/sqlite/archive_tiers/write.py @@ -6663,7 +6663,14 @@ def _set_edge( conn.execute("DELETE FROM session_model_usage WHERE session_id = ?", (child_session_id,)) _aggregate_message_tokens_into_model_usage(conn, child_session_id) _aggregate_provider_usage_into_model_usage(conn, child_session_id) - conn.execute("DELETE FROM insight_materialization WHERE session_id = ?", (child_session_id,)) + # Derived session products cache the pre-extraction message set. Their + # staleness predicate compares the session's sort key and updated-at, and + # re-extraction changes neither, so the rows are dropped outright: a missing + # profile is what makes the child a convergence candidate again. + conn.execute("DELETE FROM session_profiles WHERE session_id = ?", (child_session_id,)) + conn.execute("DELETE FROM session_latency_profiles WHERE session_id = ?", (child_session_id,)) + conn.execute("DELETE FROM session_work_events WHERE session_id = ?", (child_session_id,)) + conn.execute("DELETE FROM session_phases WHERE session_id = ?", (child_session_id,)) record_substage("count_refresh", t0) diff --git a/tests/unit/storage/test_lineage_normalization.py b/tests/unit/storage/test_lineage_normalization.py index 97ed70f00a..4de83ffefb 100644 --- a/tests/unit/storage/test_lineage_normalization.py +++ b/tests/unit/storage/test_lineage_normalization.py @@ -26,7 +26,8 @@ ParsedSessionEvent, ) from polylogue.sources.parsers.hermes_state import parse_state_db -from polylogue.storage.runtime import LineageCompleteness +from polylogue.storage.derived.session.status import session_profile_repair_candidate_ids_sync +from polylogue.storage.runtime import SESSION_INSIGHT_MATERIALIZER_VERSION, LineageCompleteness from polylogue.storage.sqlite.archive_tiers import write as _write_module from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -56,7 +57,15 @@ def _connect(path: Path) -> sqlite3.Connection: return conn -def _msg(pid: str, role: Role, text: str, position: int, *, variant_index: int = 0) -> ParsedMessage: +def _msg( + pid: str, + role: Role, + text: str, + position: int, + *, + variant_index: int = 0, + timestamp: str | None = None, +) -> ParsedMessage: return ParsedMessage( provider_message_id=pid, role=role, @@ -65,10 +74,42 @@ def _msg(pid: str, role: Role, text: str, position: int, *, variant_index: int = variant_index=variant_index, is_active_path=True, is_active_leaf=False, + timestamp=timestamp, blocks=[ParsedContentBlock(type=BlockType.TEXT, text=text)], ) +def _seed_fresh_session_products(conn: sqlite3.Connection, session_id: str, *, message_count: int) -> None: + """Materialize the derived session products a converger would have written + for ``session_id`` as it stands, copying the session's own freshness + carriers so the staleness predicate reports the profile current.""" + conn.execute( + """ + INSERT INTO session_profiles ( + session_id, materializer_version, materialized_at, source_updated_at, + source_sort_key, input_content_hash, input_row_count, source_name, + message_count, work_event_count, phase_count + ) + SELECT session_id, ?, '', datetime(updated_at_ms / 1000, 'unixepoch'), + CAST(sort_key_ms AS REAL) / 1000.0, lower(hex(content_hash)), ?, origin, ?, 1, 1 + FROM sessions + WHERE session_id = ? + """, + (SESSION_INSIGHT_MATERIALIZER_VERSION, message_count, message_count, session_id), + ) + conn.execute( + "INSERT INTO session_latency_profiles (session_id, materializer_version, materialized_at, source_name)" + " VALUES (?, ?, '', '')", + (session_id, SESSION_INSIGHT_MATERIALIZER_VERSION), + ) + conn.execute( + "INSERT INTO session_work_events (session_id, position, work_event_type, summary)" + " VALUES (?, 0, 'edit', 'seeded')", + (session_id,), + ) + conn.execute("INSERT INTO session_phases (session_id, position) VALUES (?, 0)", (session_id,)) + + async def _read_texts(path: Path, session_id: str) -> list[str | None]: conn = await aiosqlite.connect(path) try: @@ -401,6 +442,61 @@ def test_child_before_parent_is_reextracted_on_resolution(tmp_path: Path) -> Non assert composed == ["hello", "hi there", "child diverges here", "child reply"] +def test_late_parent_resolution_invalidates_child_derived_products(tmp_path: Path) -> None: + """Re-extraction drops the child's derived session products. + + Their staleness predicate compares the session's sort key, updated-at and + content hash, and re-extraction moves none of the three, so a profile + materialized over the whole child would report fresh forever. Anti-vacuity: + the seeded profile is fresh by construction (asserted before the parent + arrives), so dropping the invalidating deletes from + ``_reextract_prefix_tail_db`` leaves the stale rows in place and the child + out of the repair-candidate set. + """ + db = tmp_path / "index.db" + conn = _connect(db) + + child = ParsedSession( + source_name=Provider.CODEX, + provider_session_id="child", + title="child", + parent_session_provider_id="parent", + branch_type=BranchType.FORK, + messages=[ + _msg("c0", Role.USER, "hello", 0, timestamp="2026-01-01T00:00:00+00:00"), + _msg("c1", Role.ASSISTANT, "hi there", 1, timestamp="2026-01-01T00:01:00+00:00"), + _msg("cx", Role.USER, "child diverges here", 2, timestamp="2026-01-01T00:02:00+00:00"), + _msg("cy", Role.ASSISTANT, "child reply", 3, timestamp="2026-01-01T00:03:00+00:00"), + ], + ) + child_id = write_parsed_session_to_archive(conn, child) + _seed_fresh_session_products(conn, child_id, message_count=4) + assert child_id not in session_profile_repair_candidate_ids_sync(conn) + + parent = ParsedSession( + source_name=Provider.CODEX, + provider_session_id="parent", + title="parent", + messages=[ + _msg("p0", Role.USER, "hello", 0, timestamp="2026-01-01T00:00:00+00:00"), + _msg("p1", Role.ASSISTANT, "hi there", 1, timestamp="2026-01-01T00:01:00+00:00"), + _msg("p2", Role.USER, "parent continues alone", 2, timestamp="2026-01-01T00:05:00+00:00"), + ], + ) + write_parsed_session_to_archive(conn, parent) + + stored = conn.execute( + "SELECT position FROM messages WHERE session_id = ? ORDER BY position", (child_id,) + ).fetchall() + assert [row[0] for row in stored] == [2, 3] + for relation in ("session_profiles", "session_latency_profiles", "session_work_events", "session_phases"): + retained = conn.execute(f"SELECT COUNT(*) FROM {relation} WHERE session_id = ?", (child_id,)).fetchone()[0] + assert retained == 0, f"{relation} retained the pre-extraction projection" + assert child_id in session_profile_repair_candidate_ids_sync(conn) + + conn.close() + + @pytest.mark.parametrize("child_first", [False, True], ids=["parent-first", "child-first"]) def test_variant_prefix_lineage_converges_across_order_and_parent_replacement( tmp_path: Path, child_first: bool diff --git a/tests/unit/storage/test_session_insight_status_descriptors.py b/tests/unit/storage/test_session_insight_status_descriptors.py index 11a4121bd1..5d3df47cd9 100644 --- a/tests/unit/storage/test_session_insight_status_descriptors.py +++ b/tests/unit/storage/test_session_insight_status_descriptors.py @@ -2,13 +2,18 @@ from __future__ import annotations +import dataclasses import sqlite3 from dataclasses import asdict from pathlib import Path import aiosqlite +from polylogue.storage.derived.session.runtime import SessionInsightStatusSnapshot from polylogue.storage.derived.session.status import ( + _COUNT_DESCRIPTORS, + _FTS_DESCRIPTORS, + _TABLE_DESCRIPTORS, SessionInsightCountDescriptor, SessionInsightFtsDescriptor, session_insight_status_async, @@ -16,6 +21,8 @@ session_profile_repair_candidate_ids_sync, ) from polylogue.storage.runtime import SESSION_INSIGHT_MATERIALIZER_VERSION +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier def test_fts_descriptor_reports_duplicate_counts() -> None: @@ -366,3 +373,37 @@ async def test_lightweight_status_sync_and_async_match_with_freshness_tables(tmp # populated by any readiness descriptor (the merged-fts index is now # tracked via session_work_event_fts). #944 follow-up wires the descriptor. assert sync_status.profile_merged_fts_duplicate_count == 0 # not yet populated + + +def test_status_descriptors_resolve_against_declared_tables_and_snapshot_fields() -> None: + """Descriptor keys are resolved by subscript, so a descriptor naming a + relation or a count that no longer exists fails every status call rather + than import. Anti-vacuity: reinstating a descriptor keyed on a retired + table, or emitting a count the snapshot does not declare, turns this red. + """ + declared_tables = {descriptor.key for descriptor in _TABLE_DESCRIPTORS} + snapshot_fields = {field.name for field in dataclasses.fields(SessionInsightStatusSnapshot)} + + emitted = {descriptor.count_key for descriptor in _TABLE_DESCRIPTORS if descriptor.count_key is not None} + emitted |= {descriptor.count_key for descriptor in _COUNT_DESCRIPTORS} + emitted |= {descriptor.count_key for descriptor in _FTS_DESCRIPTORS} + emitted |= {descriptor.duplicate_count_key for descriptor in _FTS_DESCRIPTORS} + + referenced = {descriptor.table_key for descriptor in _FTS_DESCRIPTORS} + referenced |= {descriptor.table_key for descriptor in _COUNT_DESCRIPTORS if descriptor.table_key is not None} + + assert referenced <= declared_tables + assert emitted <= snapshot_fields + + +def test_status_snapshot_reads_a_fresh_index_tier(tmp_path: Path) -> None: + """The status route runs against real index DDL. Anti-vacuity: a descriptor + that queries a relation the index tier does not create raises + ``sqlite3.OperationalError`` here. + """ + with sqlite3.connect(tmp_path / "index.db") as conn: + initialize_archive_tier(conn, ArchiveTier.INDEX) + status = session_insight_status_sync(conn) + + assert status.total_sessions == 0 + assert status.missing_profile_row_count == 0 From abd32c3a354e42877f70ab868f4f2b69b769a1da Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 20:45:55 +0200 Subject: [PATCH 17/47] fix: Gate thread and tag-rollup status counts on their product tables #4649 removed the readiness descriptor whose `zero_counts` short-circuited `stale_thread_count`, `orphan_thread_count` and `stale_tag_rollup_count` when their product tables were absent, leaving the three count descriptors querying `session_profiles`, `threads` and `session_tag_rollups` unguarded. Restore the gates as `table_key`, so a status call on an archive without those relations falls back to zero instead of raising `no such table`. Co-Authored-By: Claude Opus 5 --- polylogue/storage/derived/session/status.py | 3 +++ polylogue/storage/sqlite/archive_tiers/write.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/polylogue/storage/derived/session/status.py b/polylogue/storage/derived/session/status.py index 9bd3f25c90..d2727733b5 100644 --- a/polylogue/storage/derived/session/status.py +++ b/polylogue/storage/derived/session/status.py @@ -498,12 +498,14 @@ async def _stale_session_profile_count_sql_async(conn: aiosqlite.Connection) -> ), SessionInsightCountDescriptor( count_key="stale_thread_count", + table_key="session_profiles", sql=STALE_THREAD_COUNT_SQL, params=(SESSION_INSIGHT_MATERIALIZER_VERSION,), requires_freshness=True, ), SessionInsightCountDescriptor( count_key="orphan_thread_count", + table_key="threads", sql=ORPHAN_THREAD_COUNT_SQL, requires_freshness=True, ), @@ -516,6 +518,7 @@ async def _stale_session_profile_count_sql_async(conn: aiosqlite.Connection) -> ), SessionInsightCountDescriptor( count_key="stale_tag_rollup_count", + table_key="session_tag_rollups", sql="SELECT COUNT(*) FROM session_tag_rollups WHERE materialized_at != 'query-time'", requires_freshness=True, ), diff --git a/polylogue/storage/sqlite/archive_tiers/write.py b/polylogue/storage/sqlite/archive_tiers/write.py index 4fd60ad874..85f9f60d66 100644 --- a/polylogue/storage/sqlite/archive_tiers/write.py +++ b/polylogue/storage/sqlite/archive_tiers/write.py @@ -6664,9 +6664,10 @@ def _set_edge( _aggregate_message_tokens_into_model_usage(conn, child_session_id) _aggregate_provider_usage_into_model_usage(conn, child_session_id) # Derived session products cache the pre-extraction message set. Their - # staleness predicate compares the session's sort key and updated-at, and - # re-extraction changes neither, so the rows are dropped outright: a missing - # profile is what makes the child a convergence candidate again. + # staleness predicate compares the session's sort key, updated-at and + # content hash, none of which re-extraction moves, so the rows are dropped + # outright: a missing profile is what makes the child a convergence + # candidate again. conn.execute("DELETE FROM session_profiles WHERE session_id = ?", (child_session_id,)) conn.execute("DELETE FROM session_latency_profiles WHERE session_id = ?", (child_session_id,)) conn.execute("DELETE FROM session_work_events WHERE session_id = ?", (child_session_id,)) From b805ee2d300a7c1375227145b452cec47605b7f8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 21:28:40 +0200 Subject: [PATCH 18/47] fix: Declare the read guard's degradable schema objects The readable index guard admitted any object whose name starts with `messages_fts`, so an undeclared object sharing that prefix was survivable before anyone decided it was. Declare the five objects the message FTS surface owns and match membership. `search_archive_blocks` reached SQL against that admitted-absent surface and raised `no such table: messages_fts`; it now refuses with the same typed `DatabaseError` the other search routes raise. Presence is the check, not freshness: rebuild and differential routes read this surface while it is legitimately behind `blocks`. Deletes `_readable_without_fts`, an unreachable copy of the same admission decision. Co-Authored-By: Claude Opus 5 --- .../storage/sqlite/archive_tiers/write.py | 16 +++- polylogue/storage/sqlite/schema.py | 11 --- polylogue/storage/sqlite/schema_manifest.py | 26 +++++-- .../storage/test_schema_policy_contracts.py | 77 ++++++++++++++++++- 4 files changed, 110 insertions(+), 20 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/write.py b/polylogue/storage/sqlite/archive_tiers/write.py index 85f9f60d66..f06c8c7e42 100644 --- a/polylogue/storage/sqlite/archive_tiers/write.py +++ b/polylogue/storage/sqlite/archive_tiers/write.py @@ -1852,10 +1852,24 @@ def read_session_agent_policies(conn: sqlite3.Connection, session_id: str) -> li def search_archive_blocks(conn: sqlite3.Connection, query: str) -> list[str]: - """Return block ids matched by the archive contentless FTS table.""" + """Return block ids matched by the archive contentless FTS table. + + A read open admits an index whose message FTS surface is absent + (``MESSAGE_FTS_DEGRADABLE_OBJECTS``) because search reports that state + itself. Honour that here rather than reaching SQL and raising + ``no such table: messages_fts``. Presence, not freshness, is the check: + rebuild and differential routes read this surface mid-convergence, when + it is legitimately behind ``blocks``. + """ + from polylogue.core.errors import DatabaseError + from polylogue.storage.fts.fts_lifecycle import MESSAGE_SEARCH_REPAIR_HINT + from polylogue.storage.introspection import table_exists + match_query = normalize_fts5_query(query) if match_query is None: return [] + if not table_exists(conn, "messages_fts"): + raise DatabaseError(f"Search index not built. {MESSAGE_SEARCH_REPAIR_HINT}") conn.row_factory = sqlite3.Row rows = conn.execute( """ diff --git a/polylogue/storage/sqlite/schema.py b/polylogue/storage/sqlite/schema.py index e337257ca0..d082d9af27 100644 --- a/polylogue/storage/sqlite/schema.py +++ b/polylogue/storage/sqlite/schema.py @@ -114,17 +114,6 @@ def assert_readable_archive_layout(conn: sqlite3.Connection, *, generation_id: s assert_derived_schema_identity(conn, "index") -def _readable_without_fts(conn: sqlite3.Connection) -> bool: - """Allow read-only structured access when only message FTS is missing.""" - rows = conn.execute("SELECT type, name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'").fetchall() - names = {(str(kind), str(name)) for kind, name in rows} - expected = canonical_schema_manifest(ArchiveTier.INDEX).objects - missing = {(kind, name) for kind, name, _ in expected} - names - return bool(missing) and all( - name.startswith("messages_fts") or name.startswith("blocks_command_trigram") for _kind, name in missing - ) - - def _ensure_schema(conn: sqlite3.Connection) -> None: """Ensure the database is at the current schema version. diff --git a/polylogue/storage/sqlite/schema_manifest.py b/polylogue/storage/sqlite/schema_manifest.py index 0e43ea951e..7451b13aba 100644 --- a/polylogue/storage/sqlite/schema_manifest.py +++ b/polylogue/storage/sqlite/schema_manifest.py @@ -166,8 +166,22 @@ def schema_manifest_diff(expected: SchemaManifest, actual: SchemaManifest) -> di #: The message FTS surface is a derived read model inside the derived index: #: contentless, trigger-maintained, and rebuildable from ``blocks``. Its -#: absence degrades search; the rest of the index stays readable. -MESSAGE_FTS_OBJECT_PREFIX = "messages_fts" +#: absence degrades search; the rest of the index stays readable, so a read +#: open admits a manifest diff confined to these objects. +#: +#: Membership is declared, not matched on the ``messages_fts`` name prefix. +#: ``messages_fts_identity`` is a plain declared table of ours rather than +#: FTS5 storage, and a later object sharing the prefix would otherwise be +#: admitted before anyone decided its absence is survivable. +MESSAGE_FTS_DEGRADABLE_OBJECTS: frozenset[tuple[str, str]] = frozenset( + { + ("table", "messages_fts"), + ("table", "messages_fts_identity"), + ("trigger", "messages_fts_ai"), + ("trigger", "messages_fts_ad"), + ("trigger", "messages_fts_au"), + } +) def schema_manifest_diff_is_message_fts_only(diff: Mapping[str, object]) -> bool: @@ -175,7 +189,7 @@ def schema_manifest_diff_is_message_fts_only(diff: Mapping[str, object]) -> bool if diff.get("version"): return False - names: list[str] = [] + objects: list[tuple[str, str]] = [] for key in ("missing", "extra", "wrong_definition"): entries = diff.get(key) if not isinstance(entries, (list, tuple)): @@ -183,8 +197,8 @@ def schema_manifest_diff_is_message_fts_only(diff: Mapping[str, object]) -> bool for entry in entries: if not isinstance(entry, (list, tuple)) or len(entry) < 2: return False - names.append(str(entry[1])) - return bool(names) and all(name.startswith(MESSAGE_FTS_OBJECT_PREFIX) for name in names) + objects.append((str(entry[0]), str(entry[1]))) + return bool(objects) and all(entry in MESSAGE_FTS_DEGRADABLE_OBJECTS for entry in objects) def assert_schema_manifest(conn: sqlite3.Connection, tier: ArchiveTier) -> SchemaManifest: @@ -199,7 +213,7 @@ def assert_schema_manifest(conn: sqlite3.Connection, tier: ArchiveTier) -> Schem __all__ = [ - "MESSAGE_FTS_OBJECT_PREFIX", + "MESSAGE_FTS_DEGRADABLE_OBJECTS", "SchemaManifest", "assert_schema_manifest", "canonical_schema_manifest", diff --git a/tests/unit/storage/test_schema_policy_contracts.py b/tests/unit/storage/test_schema_policy_contracts.py index ddc0959463..497a614e0a 100644 --- a/tests/unit/storage/test_schema_policy_contracts.py +++ b/tests/unit/storage/test_schema_policy_contracts.py @@ -26,7 +26,9 @@ import pytest -from polylogue.core.errors import SchemaVersionMismatchError +from polylogue.core.enums import BlockType, Provider, Role +from polylogue.core.errors import DatabaseError, SchemaVersionMismatchError +from polylogue.sources.parsers.base_models import ParsedContentBlock, ParsedMessage, ParsedSession from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -41,7 +43,12 @@ decide_schema_bootstrap, schema_version_mismatch_message, ) -from polylogue.storage.sqlite.schema_manifest import canonical_schema_manifest, schema_manifest_diff +from polylogue.storage.sqlite.schema_manifest import ( + MESSAGE_FTS_DEGRADABLE_OBJECTS, + canonical_schema_manifest, + schema_manifest_diff, + schema_manifest_diff_is_message_fts_only, +) # --------------------------------------------------------------------------- # Canonical FTS triggers — see docs/internals.md @@ -604,3 +611,69 @@ def test_fresh_init_creates_canonical_fts_trigger_set(tmp_path: Path) -> None: missing = _CANONICAL_FTS_TRIGGERS - triggers assert not missing, f"Fresh init is missing canonical FTS triggers: {sorted(missing)}" assert triggers == _CANONICAL_FTS_TRIGGERS + + +def test_message_fts_admission_is_a_declared_object_set() -> None: + """The degradable surface is declared, not inferred from an object-name prefix. + + ``messages_fts_identity`` is a plain declared table of ours that happens to + share the ``messages_fts`` prefix; a future surface could too. Admission + names the objects it admits so a new one is refused until it is declared. + + Anti-vacuity: matching on the ``messages_fts`` name prefix admits the + undeclared object below and turns this red. + """ + diff = { + "missing": [("table", "messages_fts_speculative_surface")], + "extra": [], + "wrong_definition": [], + } + assert not schema_manifest_diff_is_message_fts_only(diff) + + declared = { + ("table", "messages_fts"), + ("table", "messages_fts_identity"), + ("trigger", "messages_fts_ai"), + ("trigger", "messages_fts_ad"), + ("trigger", "messages_fts_au"), + } + assert set(MESSAGE_FTS_DEGRADABLE_OBJECTS) == declared + canonical = {(kind, name) for kind, name, _ in canonical_schema_manifest(ArchiveTier.INDEX).objects} + assert declared <= canonical + + +def test_admitted_missing_message_fts_refuses_block_search_with_a_typed_error(tmp_path: Path) -> None: + """Every reader of the admitted-absent surface degrades with a typed error. + + ``assert_readable_archive_layout`` admits an index whose message FTS + surface is gone on the promise that reads report it as search-route state. + That promise holds only if no reader of ``messages_fts`` reaches SQL. + + Anti-vacuity: removing the existence check from ``search_archive_blocks`` + raises ``sqlite3.OperationalError: no such table: messages_fts`` instead, + which is not a ``DatabaseError``. + """ + root = tmp_path / "archive" + session = ParsedSession( + source_name=Provider.CODEX, + provider_session_id="fts-degraded-1", + messages=[ + ParsedMessage( + provider_message_id="m1", + role=Role.USER, + blocks=[ParsedContentBlock(type=BlockType.TEXT, text="one searchable needle")], + ) + ], + ) + with ArchiveStore(root) as writer: + writer.write_parsed(session) + + with sqlite3.connect(root / "index.db") as conn: + for trigger in ("messages_fts_ai", "messages_fts_ad", "messages_fts_au"): + conn.execute(f"DROP TRIGGER {trigger}") + conn.execute("DROP TABLE messages_fts") + conn.commit() + + with ArchiveStore.open_existing(root, read_only=True) as store: + with pytest.raises(DatabaseError, match="Search index"): + store.search_blocks("needle") From 7ea964a54e5cd1cb54b4f99058c1537f7d6c79c9 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 22:28:28 +0200 Subject: [PATCH 19/47] fix(storage): apply the chain reduction to replay summary events A tail-only replay (the live-append path) wrote the newly accepted chunk and left the prefix's `claude_parse_coverage` row at its own position and timestamp, while `sessions.content_hash` described the chain's reduction -- one event carrying the chain's totals, the chain's newest timestamp, and a slot after every point-in-conversation event. `_reconcile_chain_summary_events` now reconciles the retained row on all three axes the hash covers, reusing the writer's own payload, summary, timestamp, and next-position helpers so the row matches what a full replace of the composed session writes. Co-Authored-By: Claude Opus 5 --- .../archive_tiers/revision_governance.py | 33 +++- tests/unit/storage/test_revision_replay.py | 170 ++++++++++++++++++ 2 files changed, 197 insertions(+), 6 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/revision_governance.py b/polylogue/storage/sqlite/archive_tiers/revision_governance.py index 55e7f57260..e8a1ff4ac2 100644 --- a/polylogue/storage/sqlite/archive_tiers/revision_governance.py +++ b/polylogue/storage/sqlite/archive_tiers/revision_governance.py @@ -190,7 +190,11 @@ from polylogue.storage.sqlite.archive_tiers.write import ( ArchiveWriteOutcome, PreparedSessionRows, + _event_summary, + _json_dumps, + _next_session_event_position, _repair_stale_session_observations, + _timestamp_ms, replace_parser_ingest_flag_tags, upsert_parser_ingest_flag_tags, write_parsed_session_to_archive, @@ -2822,10 +2826,20 @@ def _reconcile_chain_summary_events( session_id: str, aggregate: ParsedSession, ) -> None: - """Leave one whole-input summary row per type, carrying the chain's totals.""" - for event_type in _CHAIN_SUMMARY_EVENT_TYPES: - composed = [event for event in aggregate.session_events if event.event_type == event_type] - if not composed: + """Store each whole-input summary event as the chain's content hash describes it. + + ``merge_parsed_session_chunks`` reduces a declared summary type to one + event carrying the chain's totals, the chain's newest timestamp, and a + slot after every point-in-conversation event; that reduction is what + ``aggregate_content_hash`` covers. A tail-only write appends the newly + accepted chunk's own events and leaves the prefix's chunk-local row + where it is, so reduce the stored row on all three axes. Iterating the + aggregate's own event order keeps two summary types in the order the + reduction gave them. + """ + for composed in aggregate.session_events: + event_type = composed.event_type + if event_type not in _CHAIN_SUMMARY_EVENT_TYPES: continue positions = [ int(row[0]) @@ -2841,9 +2855,16 @@ def _reconcile_chain_summary_events( (session_id, event_type, positions[-1]), ) store._conn.execute( - "UPDATE session_events SET payload_json = ? WHERE session_id = ? AND event_type = ? AND position = ?", + """ + UPDATE session_events + SET payload_json = ?, summary = ?, occurred_at_ms = ?, position = ? + WHERE session_id = ? AND event_type = ? AND position = ? + """, ( - json.dumps(composed[-1].payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")), + _json_dumps(composed.payload), + _event_summary(composed) or "", + _timestamp_ms(composed.timestamp), + _next_session_event_position(store._conn, session_id), session_id, event_type, positions[-1], diff --git a/tests/unit/storage/test_revision_replay.py b/tests/unit/storage/test_revision_replay.py index 13fcc4163b..eb11fc116f 100644 --- a/tests/unit/storage/test_revision_replay.py +++ b/tests/unit/storage/test_revision_replay.py @@ -29,6 +29,7 @@ ) from polylogue.core.enums import Provider from polylogue.core.raw_failure_evidence import RawFailureEvidenceKind +from polylogue.core.timestamp_authority import timestamp_millis from polylogue.pipeline.ids import session_content_hash, session_revision_projection from polylogue.sources.dispatch import merge_parsed_session_chunks, parse_stream_payload from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession, ParsedSessionEvent @@ -2761,3 +2762,172 @@ def parsed(*, message_id: str, text: str, seen: int) -> ParsedSession: ).fetchone() assert stored_hash is not None assert bytes(stored_hash[0]).hex() == session_content_hash(composed[0]) + + +def test_tail_only_replay_stores_the_chain_reduction_not_the_prefix_summary_row(tmp_path: Path) -> None: + """polylogue-ylrba: the live-append path persists the reduced event projection. + + ``sources/live/append_ingest.py`` replays with ``skip_already_applied``, so + only the newly accepted tail is written while the prefix's + ``claude_parse_coverage`` row -- a complete-input summary -- is already + stored. ``sessions.content_hash`` describes the chain's reduction, which + carries the chain's totals, the chain's newest timestamp, and a position + after every point-in-conversation event. All three must be what the index + holds. + + Anti-vacuity: reconciling the summary row's payload alone leaves it at the + prefix's position (before the tail's ``claude_session_kind`` event) and + stamped with the prefix's ``updated_at``, so the persisted projection + differs from the reduction the stored hash describes in both order and + timestamp, and this test is red. + """ + initialize_active_archive_root(tmp_path) + + baseline_chunk = ParsedSession( + source_name=Provider.CLAUDE_CODE, + provider_session_id="chat", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + messages=[ + ParsedMessage(provider_message_id="m0", role=Role.USER, text="zero", timestamp="2026-01-01T00:00:00Z") + ], + session_events=[ + ParsedSessionEvent( + event_type="claude_parse_coverage", + timestamp="2026-01-01T00:00:00Z", + payload={ + "sidecar_seen": {"user": 1}, + "sidecar_persisted": {"user": 1}, + "empty_dropped_by_record_type": {}, + }, + ) + ], + ) + # A tail that carries no coverage of its own is the ordinary live append: + # the parser emits the event only when the chunk saw sidecar records. + append_chunk = ParsedSession( + source_name=Provider.CLAUDE_CODE, + provider_session_id="chat", + created_at="2026-01-02T00:00:00Z", + updated_at="2026-01-02T00:00:00Z", + messages=[ + ParsedMessage(provider_message_id="m1", role=Role.USER, text="one", timestamp="2026-01-02T00:00:00Z") + ], + session_events=[ + ParsedSessionEvent( + event_type="claude_session_kind", + timestamp="2026-01-02T00:00:00Z", + payload={"session_kind": "resume"}, + ) + ], + ) + + def stored_event_projection(archive: ArchiveStore, session_id: str) -> list[tuple[str, int | None, object]]: + return [ + (str(row[0]), None if row[1] is None else int(row[1]), json.loads(str(row[2]))) + for row in archive._conn.execute( + "SELECT event_type, occurred_at_ms, payload_json FROM session_events" + " WHERE session_id = ? ORDER BY position", + (session_id,), + ).fetchall() + ] + + with ArchiveStore.open_existing(tmp_path, read_only=False) as archive: + baseline = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, payload=b"a" * 10, source_path="chat.jsonl", acquired_at_ms=1 + ) + archive.bind_raw_revision( + baseline, + RawRevisionEnvelope( + "claude-code-session:chat", + RawRevisionKind.FULL, + "full-0", + 0, + authority=RawRevisionAuthority.BYTE_PROVEN, + ), + ) + archive.apply_raw_revision_replay( + archive.classify_raw_revision_cohort_for_live_watch("claude-code-session:chat"), + {baseline: baseline_chunk}, + acquired_at_ms=0, + skip_already_applied=True, + ) + + append_one = archive.write_raw_payload( + provider=Provider.CLAUDE_CODE, + payload=b"b" * 5, + source_path="chat.jsonl", + source_index=-1, + acquired_at_ms=2, + ) + archive.bind_raw_revision( + append_one, + RawRevisionEnvelope( + "claude-code-session:chat", + RawRevisionKind.APPEND, + append_source_revision("full-0", hashlib.sha256(b"b" * 5).hexdigest()), + 1, + predecessor_source_revision="full-0", + predecessor_raw_id=baseline, + baseline_raw_id=baseline, + append_start_offset=10, + append_end_offset=15, + authority=RawRevisionAuthority.BYTE_PROVEN, + ), + ) + plan = archive.classify_raw_revision_cohort_for_live_watch("claude-code-session:chat") + assert plan.accepted_raw_ids == (baseline, append_one) + session_id, _ = archive.apply_raw_revision_replay( + plan, + {baseline: baseline_chunk, append_one: append_chunk}, + acquired_at_ms=0, + skip_already_applied=True, + ) + + composed = merge_parsed_session_chunks([baseline_chunk, append_chunk]) + assert len(composed) == 1 + expected_events = [ + (event.event_type, timestamp_millis(event.timestamp), event.payload) for event in composed[0].session_events + ] + assert stored_event_projection(archive, session_id) == expected_events + + stored_hash = archive._conn.execute( + "SELECT content_hash FROM sessions WHERE session_id = ?", + (session_id,), + ).fetchone() + assert stored_hash is not None + assert bytes(stored_hash[0]).hex() == session_content_hash(composed[0]) + + # Raw provenance and the accepted-chain receipts survive the projection fix. + head = archive._conn.execute( + "SELECT accepted_raw_id, session_id FROM raw_revision_heads WHERE logical_source_key = ?", + ("claude-code-session:chat",), + ).fetchone() + assert head is not None + assert (str(head[0]), str(head[1])) == (append_one, session_id) + applied = { + (str(row[0]), str(row[1])) + for row in archive._conn.execute( + "SELECT raw_id, decision FROM raw_revision_applications WHERE logical_source_key = ?", + ("claude-code-session:chat",), + ).fetchall() + } + assert applied == { + (baseline, ApplicationDecision.SELECTED_BASELINE.value), + (append_one, ApplicationDecision.APPLIED_APPEND.value), + } + + # Replaying the same accepted chain again changes nothing. + archive.apply_raw_revision_replay( + archive.classify_raw_revision_cohort_for_live_watch("claude-code-session:chat"), + {baseline: baseline_chunk, append_one: append_chunk}, + acquired_at_ms=0, + skip_already_applied=True, + ) + assert stored_event_projection(archive, session_id) == expected_events + replayed_hash = archive._conn.execute( + "SELECT content_hash FROM sessions WHERE session_id = ?", + (session_id,), + ).fetchone() + assert replayed_hash is not None + assert bytes(replayed_hash[0]) == bytes(stored_hash[0]) From 1dc9bf2b62cfdbaa0a9fda19f0804fbaf6da841f Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 19:59:16 +0200 Subject: [PATCH 20/47] fix: Restore the sessions record projection to SESSIONS_SPEC Every `sessions` column carried `record_name=None`, so the record projection rendered empty and `get_session` and `list_sessions` both emitted `SELECT FROM sessions`. Declare each column's record name and select expression on the column itself rather than through a separate wrapper applied after the spec, and cover the projection with a spec contract and a read-path test. Co-Authored-By: Claude Opus 5 --- .../archive_tiers/archive_tiers_specs.py | 87 ++++++++++++++++--- .../storage/test_spec_driven_hydration.py | 79 ++++++++++++++++- 2 files changed, 149 insertions(+), 17 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/archive_tiers_specs.py b/polylogue/storage/sqlite/archive_tiers/archive_tiers_specs.py index 803851b79e..26ed691e3d 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive_tiers_specs.py +++ b/polylogue/storage/sqlite/archive_tiers/archive_tiers_specs.py @@ -367,20 +367,39 @@ def _make_blocks_spec() -> TableColumnSpec: # Index-tier table specs. Each rendered table body is sourced from these # column definitions plus its table-level constraints; indexes, triggers, # and virtual tables remain in index.py because they are not row schemas. -def _raw_column(name: str, ddl_sql: str) -> ColumnSpec: - return ColumnSpec(name=name, is_generated="GENERATED ALWAYS" in ddl_sql, ddl_sql=ddl_sql) +def _raw_column( + name: str, + ddl_sql: str, + *, + record_name: str | None = None, + select_expression: str | None = None, +) -> ColumnSpec: + """Declare a stored column and, where it has one, its record projection. + + ``record_name`` is the label the record mapper consumes; a column without + one is storage-only and never reaches a runtime record. + """ + return ColumnSpec( + name=name, + is_generated="GENERATED ALWAYS" in ddl_sql, + ddl_sql=ddl_sql, + record_name=record_name, + select_expression=select_expression, + ) def _make_table_spec( table_name: str, columns: tuple[ColumnSpec, ...], *, + record_only_columns: tuple[ColumnSpec, ...] = (), table_constraints: tuple[str, ...] = (), ) -> TableColumnSpec: return TableColumnSpec( table_name=table_name, all_columns=columns, writable_columns=tuple(column for column in columns if not column.is_generated), + record_only_columns=record_only_columns, table_constraints=table_constraints, ) @@ -534,16 +553,23 @@ def _make_table_spec( _raw_column( "session_id", """session_id TEXT GENERATED ALWAYS AS (origin || ':' || native_id) STORED UNIQUE""", + record_name="session_id", ), - _raw_column("native_id", """native_id TEXT NOT NULL"""), - _raw_column("origin", f"""origin TEXT NOT NULL CHECK ({check("origin", Origin)})"""), + _raw_column("native_id", """native_id TEXT NOT NULL""", record_name="native_id"), _raw_column( - "parent_session_id", """parent_session_id TEXT REFERENCES sessions(session_id) ON DELETE SET NULL""" + "origin", + f"""origin TEXT NOT NULL CHECK ({check("origin", Origin)})""", + record_name="origin", + ), + _raw_column( + "parent_session_id", + """parent_session_id TEXT REFERENCES sessions(session_id) ON DELETE SET NULL""", + record_name="parent_session_id", ), _raw_column( "root_session_id", """root_session_id TEXT REFERENCES sessions(session_id) ON DELETE SET NULL""" ), - _raw_column("raw_id", """raw_id TEXT"""), + _raw_column("raw_id", """raw_id TEXT""", record_name="raw_id"), _raw_column( "parser_fingerprint", """-- Written by the parsed-session chokepoint in the same transaction as @@ -552,13 +578,16 @@ def _make_table_spec( ), _raw_column("lowering_fingerprint", """lowering_fingerprint TEXT"""), _raw_column( - "branch_type", f"""branch_type TEXT CHECK ({nullable_check("branch_type", BranchType)})""" + "branch_type", + f"""branch_type TEXT CHECK ({nullable_check("branch_type", BranchType)})""", + record_name="branch_type", ), _raw_column("active_leaf_message_id", """active_leaf_message_id TEXT"""), - _raw_column("title", """title TEXT"""), + _raw_column("title", """title TEXT""", record_name="title"), _raw_column( "session_kind", f"""session_kind TEXT NOT NULL DEFAULT 'standard' CHECK ({check("session_kind", SessionKind)})""", + record_name="session_kind", ), _raw_column( "title_source", @@ -593,6 +622,7 @@ def _make_table_spec( -- resolved title): this is a display label for the session's identity, -- not its content. display_name TEXT""", + record_name="display_name", ), _raw_column( "run_settings_json", @@ -603,6 +633,7 @@ def _make_table_spec( -- into typed columns would couple this schema to one provider for no -- query benefit; nothing here is queried across origins today. run_settings_json TEXT CHECK ({json_object_check("run_settings_json", nullable=True)})""", + record_name="run_settings_json", ), _raw_column( "pending_drafts_json", @@ -617,10 +648,11 @@ def _make_table_spec( -- and polylogue-nuec were fixed for, on a third axis (mutable session -- state rather than acquisition state or provider-remeasurement). pending_drafts_json TEXT CHECK ({json_array_check("pending_drafts_json", nullable=True)})""", + record_name="pending_drafts_json", ), - _raw_column("git_branch", """git_branch TEXT"""), - _raw_column("git_repository_url", """git_repository_url TEXT"""), - _raw_column("provider_project_ref", """provider_project_ref TEXT"""), + _raw_column("git_branch", """git_branch TEXT""", record_name="git_branch"), + _raw_column("git_repository_url", """git_repository_url TEXT""", record_name="git_repository_url"), + _raw_column("provider_project_ref", """provider_project_ref TEXT""", record_name="provider_project_ref"), _raw_column("commit_hash", """commit_hash TEXT"""), _raw_column("instructions_text", """instructions_text TEXT"""), _raw_column( @@ -640,6 +672,7 @@ def _make_table_spec( -- table's header) -- this column is a parallel exact-dollar figure, not -- a token source. reported_cost_usd REAL CHECK(reported_cost_usd IS NULL OR reported_cost_usd >= 0)""", + record_name="reported_cost_usd", ), _raw_column( "message_count", """message_count INTEGER NOT NULL DEFAULT 0 CHECK(message_count >= 0)""" @@ -683,12 +716,38 @@ def _make_table_spec( "assistant_word_count", """assistant_word_count INTEGER NOT NULL DEFAULT 0 CHECK(assistant_word_count >= 0)""", ), - _raw_column("content_hash", f"""content_hash BLOB NOT NULL {CONTENT_HASH_CHECK}"""), - _raw_column("created_at_ms", """created_at_ms INTEGER"""), - _raw_column("updated_at_ms", """updated_at_ms INTEGER"""), + _raw_column( + "content_hash", + f"""content_hash BLOB NOT NULL {CONTENT_HASH_CHECK}""", + record_name="content_hash", + select_expression="lower(hex({alias}.content_hash))", + ), + _raw_column( + "created_at_ms", + """created_at_ms INTEGER""", + record_name="created_at", + select_expression="datetime({alias}.created_at_ms / 1000, 'unixepoch')", + ), + _raw_column( + "updated_at_ms", + """updated_at_ms INTEGER""", + record_name="updated_at", + select_expression="datetime({alias}.updated_at_ms / 1000, 'unixepoch')", + ), _raw_column( "sort_key_ms", """sort_key_ms INTEGER GENERATED ALWAYS AS (COALESCE(updated_at_ms, created_at_ms)) STORED""", + record_name="sort_key", + select_expression="{alias}.sort_key_ms / 1000.0", + ), + ), + record_only_columns=( + ColumnSpec("metadata", record_name="metadata", select_expression="'{{}}'"), + ColumnSpec("version", record_name="version", select_expression="1"), + ColumnSpec( + "working_directories_json", + record_name="working_directories_json", + select_expression="(SELECT json_group_array(path) FROM session_working_dirs swd WHERE swd.session_id = {alias}.session_id ORDER BY position)", ), ), table_constraints=("""PRIMARY KEY(origin, native_id)""",), diff --git a/tests/unit/storage/test_spec_driven_hydration.py b/tests/unit/storage/test_spec_driven_hydration.py index 9a6e8ae714..f57fe411db 100644 --- a/tests/unit/storage/test_spec_driven_hydration.py +++ b/tests/unit/storage/test_spec_driven_hydration.py @@ -6,12 +6,13 @@ import pytest -from polylogue.core.enums import BlockType, Provider, Role +from polylogue.core.enums import BlockType, Origin, Provider, Role from polylogue.sources.parsers.base import ParsedContentBlock, ParsedMessage, ParsedSession from polylogue.storage.hydrators import message_from_record -from polylogue.storage.sqlite.archive_tiers.archive_tiers_specs import BLOCKS_SPEC, MESSAGES_SPEC +from polylogue.storage.runtime import SessionRecord +from polylogue.storage.sqlite.archive_tiers.archive_tiers_specs import BLOCKS_SPEC, MESSAGES_SPEC, SESSIONS_SPEC from polylogue.storage.sqlite.async_sqlite import SQLiteBackend -from polylogue.storage.sqlite.queries import message_query_reads +from polylogue.storage.sqlite.queries import message_query_reads, sessions_reads from tests.infra.identity import archive_message_id from tests.infra.live_ingest import ingest_session @@ -67,3 +68,75 @@ async def test_real_archive_write_read_hydrates_spec_fields(tmp_path: Path) -> N assert hydrated.text == "finished" assert hydrated.timestamp is not None assert hydrated.timestamp.isoformat() == "2026-03-01T10:05:00+00:00" + + +def test_sessions_spec_declares_the_record_projection_the_mapper_consumes() -> None: + """SESSIONS_SPEC must name every field ``_row_to_session`` builds a record from. + + Red when any ``record_name`` is dropped from SESSIONS_SPEC: an empty + projection lowers to ``SELECT FROM sessions``, and a partial one leaves a + SessionRecord field with no column to read. + """ + projection = SESSIONS_SPEC.record_select_column_names("sessions") + assert projection.strip(), "sessions record projection is empty" + + declared = {column.record_name for column in SESSIONS_SPEC.record_columns} + # The mapper builds these two fields from the raw JSON projections rather + # than a same-named column. + from_json_projection = {"run_settings": "run_settings_json", "pending_drafts": "pending_drafts_json"} + missing = { + field + for field in SessionRecord.model_fields + if field not in declared and from_json_projection.get(field) not in declared + } + assert not missing, f"SessionRecord fields with no sessions projection: {sorted(missing)}" + + +@pytest.mark.asyncio +async def test_session_reads_hydrate_records_through_the_declared_projection(tmp_path: Path) -> None: + """``get_session`` and ``list_sessions`` return populated records. + + Red when the sessions projection is empty (both statements become invalid + SQL) and when any single projected column loses its ``record_name`` (the + corresponding record field falls back to its default). + """ + backend = SQLiteBackend(db_path=tmp_path / "index.db") + parsed_session = ParsedSession( + source_name=Provider.CLAUDE_CODE, + provider_session_id="session-projection", + title="Session projection", + git_branch="feature/projection", + messages=[ + ParsedMessage( + provider_message_id="native-message", + role=Role.USER, + text="hello", + timestamp="2026-03-01T10:05:00+00:00", + position=0, + blocks=[ParsedContentBlock(type=BlockType.TEXT, text="hello")], + ) + ], + ) + try: + session_id = await ingest_session(parsed_session, backend) + async with backend.connection() as conn: + record = await sessions_reads.get_session(conn, session_id) + listed = await sessions_reads.list_sessions(conn, limit=10) + finally: + await backend.close() + + assert record is not None + assert str(record.session_id) == session_id + assert record.origin is Origin.CLAUDE_CODE_SESSION + assert record.native_id == "session-projection" + assert record.title == "Session projection" + assert record.git_branch == "feature/projection" + assert record.content_hash + assert record.version == 1 + # The record-only projection is live: no working dirs still yields an array. + assert record.working_directories_json == "[]" + assert record.sort_key is not None + assert record.updated_at is not None + + assert [str(item.session_id) for item in listed] == [session_id] + assert listed[0].title == "Session projection" From f563965948be97bbf47a8d22552c1b0323e175a5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 19:20:42 +0200 Subject: [PATCH 21/47] fix: Preserve dispatch links across parent replacement and aliases Replacing a parent deletes its blocks, and the ON DELETE SET NULL foreign key on session_links.parent_tool_use_block_id nulls every inbound child edge while the replacement reinserts the same deterministic block ids. Identity resolution revisits only unresolved edges, so the join key stayed NULL and the dispatch dropped to unresolved. _refill_inbound_dispatch_block_ids rebinds resolved children on every write. Dispatch child identity is now read from the progress payload alone -- the record envelope names the emitting session -- and candidate names are compared as the sessions they resolve to, so several exact names for one child are one identity. Co-Authored-By: Claude Opus 5 --- .../schema-disposition-2026-08-31.json | 2 +- .../sources/parsers/claude/code_parser.py | 10 +- .../storage/sqlite/archive_tiers/index.py | 7 +- .../storage/sqlite/archive_tiers/write.py | 114 +++++++-- .../test_parsers_claude_code_artifacts.py | 27 ++ .../storage/test_unread_wire_batch_v46.py | 231 ++++++++++++++++++ 6 files changed, 366 insertions(+), 25 deletions(-) diff --git a/docs/evidence/schema-disposition-2026-08-31.json b/docs/evidence/schema-disposition-2026-08-31.json index cc75a7f737..5e342e3bfa 100644 --- a/docs/evidence/schema-disposition-2026-08-31.json +++ b/docs/evidence/schema-disposition-2026-08-31.json @@ -39859,7 +39859,7 @@ "schema_versions": { "audit": 2, "embeddings": 5, - "index": 92, + "index": 93, "ops": 1, "source": 41, "user": 11 diff --git a/polylogue/sources/parsers/claude/code_parser.py b/polylogue/sources/parsers/claude/code_parser.py index aae40b1347..7aa62af71d 100644 --- a/polylogue/sources/parsers/claude/code_parser.py +++ b/polylogue/sources/parsers/claude/code_parser.py @@ -612,11 +612,13 @@ def _accumulate_delegation_progress( if not parent_tool_use_id: return False entry = accumulator.setdefault(parent_tool_use_id, _DelegationProgressStats()) - data_map = data + # The dispatched child is named only inside the progress payload. The + # record envelope's identity fields name the transcript that emitted the + # tick -- its own session, which is the dispatching parent. for key in ("childSessionId", "child_session_id", "agentId", "agent_id"): - for value in (_string_field(item, key), _string_field(data_map, key)): - if value: - entry.child_provider_ids.add(value) + value = _string_field(data, key) + if value: + entry.child_provider_ids.add(value) entry.count += 1 if timestamp: if entry.first_seen is None or timestamp < entry.first_seen: diff --git a/polylogue/storage/sqlite/archive_tiers/index.py b/polylogue/storage/sqlite/archive_tiers/index.py index 678b6dd94a..ccb138a4ba 100644 --- a/polylogue/storage/sqlite/archive_tiers/index.py +++ b/polylogue/storage/sqlite/archive_tiers/index.py @@ -443,7 +443,12 @@ # observations. Existing indexes must be replayed to reconstruct topology. # v92 removes content and cardinality pairing from delegation projection; # existing materialized rows must be regenerated from exact session links. -INDEX_SCHEMA_VERSION = 92 +# polylogue-vid0.1: v93 reads dispatch child identity from the progress +# payload alone and compares identities as the sessions they resolve to. +# SEMANTIC_REPARSE: links materialized under v92 recorded the emitting +# session's own name as a competing child identity, and no clone-safe SQL +# delta can recover the dispatch join key those rows refused. +INDEX_SCHEMA_VERSION = 93 # polylogue-v6i3: shared WHEN-clause fragment gating the blocks_command_trigram # trigger BODIES on the same dedicated bulk-build guard row messages_fts's diff --git a/polylogue/storage/sqlite/archive_tiers/write.py b/polylogue/storage/sqlite/archive_tiers/write.py index f06c8c7e42..80f8c5a6d2 100644 --- a/polylogue/storage/sqlite/archive_tiers/write.py +++ b/polylogue/storage/sqlite/archive_tiers/write.py @@ -4512,6 +4512,9 @@ def record_substage(name: str, started_at: float) -> None: _resolve_outbound_session_links(conn, session_id, origin) record_substage("outbound_links", t0) t0 = time.perf_counter() + _refill_inbound_dispatch_block_ids(conn, session_id) + record_substage("inbound_dispatch_blocks", t0) + t0 = time.perf_counter() has_outbound_link = ( conn.execute("SELECT 1 FROM session_links WHERE src_session_id = ? LIMIT 1", (session_id,)).fetchone() is not None @@ -4617,6 +4620,38 @@ def record_substage(name: str, started_at: float) -> None: record_substage("projection_refresh", t0) +def _refill_inbound_dispatch_block_ids(conn: sqlite3.Connection, parent_session_id: str) -> None: + """Rebind resolved children to this parent's dispatch blocks. + + Writing a parent replaces its messages and blocks, and + ``session_links.parent_tool_use_block_id`` is ``ON DELETE SET NULL``, so + every inbound child edge loses the join key while the replacement + reinserts the same deterministic block ids. Identity resolution revisits + only unresolved edges, so an already-resolved child is repaired here or + not at all. + """ + rows = conn.execute( + """SELECT src_session_id, dst_origin, dst_native_id, link_type + FROM session_links + WHERE resolved_dst_session_id = ? + AND parent_tool_use_block_id IS NULL + AND status IS NULL""", + (parent_session_id,), + ).fetchall() + for src_session_id, dst_origin, dst_native_id, link_type in rows: + block_id = _resolve_parent_dispatch_block_id(conn, parent_session_id, str(src_session_id)) + if block_id is None: + continue + conn.execute( + """UPDATE session_links + SET parent_tool_use_block_id = ?, + method = CASE WHEN method = 'parser-parent' THEN 'parent-tool-use-id' ELSE method END + WHERE src_session_id = ? AND dst_origin = ? AND dst_native_id = ? AND link_type = ? + AND parent_tool_use_block_id IS NULL""", + (block_id, src_session_id, dst_origin, dst_native_id, link_type), + ) + + def _root_projection_current(conn: sqlite3.Connection, session_id: str) -> bool: row = conn.execute( """ @@ -7195,20 +7230,58 @@ def _existing_parent_session_id(conn: sqlite3.Connection, session: ParsedSession return str(row[0][0]) if len(row) == 1 else None +def _dispatch_child_identity_values(observation: ParsedDispatchObservation, payload: Mapping[str, object]) -> set[str]: + """Every exact provider name this dispatch observation offers for its child.""" + values = {observation.child_provider_id.strip()} if observation.child_provider_id else set() + candidates = payload.get("child_provider_ids") + if isinstance(candidates, list): + values |= {str(value).strip() for value in candidates if str(value).strip()} + return values + + +def _canonical_identity_session_ids(conn: sqlite3.Connection, origin: str, values: set[str]) -> set[str] | None: + """Resolve exact provider names to the sessions that claim them. + + ``None`` means at least one name is not resolvable to exactly one session + -- an unclaimed or contested name could still turn out to be a different + session, so the caller must refuse rather than assume agreement. + """ + resolved: set[str] = set() + for value in values: + claimants = { + str(row[0]) + for row in conn.execute( + """SELECT DISTINCT claimant_session_id FROM session_identity_claims + WHERE origin = ? AND identity_namespace = 'provider-session' + AND provider_value = ?""", + (origin, value), + ).fetchall() + } + if len(claimants) > 1: + return None + if claimants: + resolved |= claimants + continue + canonical_id = archive_session_id(origin, value) + if conn.execute("SELECT 1 FROM sessions WHERE session_id = ?", (canonical_id,)).fetchone() is None: + return None + resolved.add(canonical_id) + return resolved + + def _resolve_parent_dispatch_block_id( conn: sqlite3.Connection, parent_session_id: str, child_session_id: str ) -> str | None: - """Resolve parent-side delegation evidence after either session arrives.""" - child_ids = { - str(row[0]) - for row in conn.execute( - """SELECT provider_value FROM session_identity_claims - WHERE claimant_session_id = ? AND identity_namespace = 'provider-session'""", - (child_session_id,), - ).fetchall() - } - if not child_ids: + """Resolve parent-side delegation evidence after either session arrives. + + Provider names are compared as the sessions they resolve to: several exact + names for one child are one identity, and only names resolving to + different sessions contradict each other. + """ + origin_row = conn.execute("SELECT origin FROM sessions WHERE session_id = ?", (parent_session_id,)).fetchone() + if origin_row is None: return None + origin = str(origin_row[0]) rows = conn.execute( """SELECT source_message_provider_id, payload_json FROM session_events @@ -7216,7 +7289,6 @@ def _resolve_parent_dispatch_block_id( (parent_session_id,), ).fetchall() matching_blocks: set[str] = set() - matching_child_ids: set[str] = set() for source_id, payload_json in rows: try: payload = json.loads(str(payload_json)) @@ -7229,22 +7301,26 @@ def _resolve_parent_dispatch_block_id( observation = ParsedDispatchObservation.model_validate(observation_payload) except (TypeError, ValueError): continue - if observation.resolution_reason is not None: + # A parser-side contradiction is provisional: it compares raw names + # without an archive to resolve them against. Every other refusal + # reason stands. + if observation.resolution_reason is not None and observation.resolution_reason != "identity-contradiction": continue - child_provider_id = observation.child_provider_id - if child_provider_id is None or child_provider_id not in child_ids: + identity_values = _dispatch_child_identity_values(observation, observation_payload) + if not identity_values: continue - rows = conn.execute( + if _canonical_identity_session_ids(conn, origin, identity_values) != {child_session_id}: + continue + block_rows = conn.execute( """SELECT b.block_id FROM blocks b JOIN messages m ON m.message_id = b.message_id WHERE b.tool_id = ? AND b.block_type = 'tool_use' AND m.session_id = ? ORDER BY b.block_id""", (observation.provider_tool_id, parent_session_id), ).fetchall() - if len(rows) == 1: - matching_blocks.add(str(rows[0][0])) - matching_child_ids.add(child_provider_id) - if len(matching_blocks) == 1 and len(matching_child_ids) == 1: + if len(block_rows) == 1: + matching_blocks.add(str(block_rows[0][0])) + if len(matching_blocks) == 1: return next(iter(matching_blocks)) return None diff --git a/tests/unit/sources/test_parsers_claude_code_artifacts.py b/tests/unit/sources/test_parsers_claude_code_artifacts.py index 404a0c9a92..ff895fe2eb 100644 --- a/tests/unit/sources/test_parsers_claude_code_artifacts.py +++ b/tests/unit/sources/test_parsers_claude_code_artifacts.py @@ -1048,6 +1048,33 @@ def test_parse_code_quarantines_conflicting_dispatch_child_identity() -> None: assert event.payload["resolution_reason"] == "identity-contradiction" +def test_parse_code_ignores_emitting_identity_on_dispatch_progress() -> None: + """A progress record's envelope names its own session, never the child. + + Every one of the 8,282 live ``agent_progress`` records carrying a + record-level ``agentId`` repeats the emitting transcript's own identity; + the dispatched child is named only in the progress payload. Red if + envelope identity fields are folded back into the child identity set, + which turns an ordinary dispatch into an identity contradiction. + """ + records = [ + { + "type": "progress", + "uuid": "progress-a", + "sessionId": "parent-native", + "parentToolUseID": "dispatch-1", + "agentId": "parent-own-agent", + "data": {"type": "agent_progress", "agentId": "child-agent"}, + }, + ] + + parsed = parse_code(records, "agent-parent-own-agent") + [event] = [event for event in parsed.session_events if event.event_type == "claude_delegation_progress"] + assert event.payload["child_provider_id"] == "child-agent" + assert event.payload["resolution_reason"] is None + assert "child_provider_ids" not in event.payload + + def test_parse_code_drops_non_message_sidecars() -> None: items: list[object] = [ { diff --git a/tests/unit/storage/test_unread_wire_batch_v46.py b/tests/unit/storage/test_unread_wire_batch_v46.py index c31c25733a..0c21f31cba 100644 --- a/tests/unit/storage/test_unread_wire_batch_v46.py +++ b/tests/unit/storage/test_unread_wire_batch_v46.py @@ -916,3 +916,234 @@ async def test_session_identity_alias_conflict_invalidates_resolved_child(tmp_pa assert link["resolved_dst_session_id"] is None assert link["parent_session_id"] is None assert link["root_session_id"] == child_id + + +def _dispatching_parent_records(*, extra_text: str | None = None) -> list[object]: + """Provider-shaped parent transcript that dispatches one subagent.""" + records: list[object] = [ + { + "type": "assistant", + "uuid": "parent-assistant", + "sessionId": "parent-native", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "dispatch-tool-1", + "name": "Task", + "input": {"description": "spawn"}, + } + ], + }, + }, + { + "type": "progress", + "uuid": "parent-progress", + "sessionId": "parent-native", + "parentToolUseID": "dispatch-tool-1", + "data": {"type": "agent_progress", "childSessionId": "agent-child-native"}, + }, + ] + if extra_text is not None: + records.append( + { + "type": "assistant", + "uuid": "parent-assistant-2", + "sessionId": "parent-native", + "message": {"role": "assistant", "content": [{"type": "text", "text": extra_text}]}, + } + ) + return records + + +def _dispatched_child_records() -> list[object]: + return [ + { + "type": "user", + "uuid": "child-user", + "sessionId": "parent-native", + "message": {"role": "user", "content": "work"}, + } + ] + + +async def test_parent_replacement_preserves_child_dispatch_block_id(tmp_path: Path) -> None: + """Replacing a resolved parent keeps the child's dispatch join key. + + Full replacement deletes the parent's blocks, and the ON DELETE SET NULL + foreign key nulls every inbound ``parent_tool_use_block_id`` while the + replacement reinserts the same deterministic block ids. Red if the writer + only refills that column for edges whose session identity is still + unresolved. + """ + backend = SQLiteBackend(db_path=tmp_path / "parent-replacement-dispatch.db") + repo = SessionRepository(backend=backend) + try: + parent_id = await ingest_session(parse_code(_dispatching_parent_records(), "parent-file"), backend=backend) + child_id = await ingest_session(parse_code(_dispatched_child_records(), "agent-child-native"), backend=backend) + + async with backend.connection() as conn: + [before] = await conn.execute_fetchall( + """SELECT resolved_dst_session_id, parent_tool_use_block_id, method + FROM session_links WHERE src_session_id = ?""", + (child_id,), + ) + + await ingest_session( + parse_code(_dispatching_parent_records(extra_text="second turn"), "parent-file"), + backend=backend, + ) + + async with backend.connection() as conn: + [after] = await conn.execute_fetchall( + """SELECT resolved_dst_session_id, parent_tool_use_block_id, method + FROM session_links WHERE src_session_id = ?""", + (child_id,), + ) + [fact] = await conn.execute_fetchall( + """SELECT mapping_state, child_session_id FROM delegation_facts + WHERE parent_session_id = ?""", + (parent_id,), + ) + finally: + await repo.close() + + dispatch_block_id = archive_block_id(archive_message_id(parent_id, "parent-assistant", position=0), position=0) + assert before["parent_tool_use_block_id"] == dispatch_block_id + assert after["resolved_dst_session_id"] == parent_id + assert after["parent_tool_use_block_id"] == dispatch_block_id + assert after["method"] == "parent-tool-use-id" + assert fact["mapping_state"] == "resolved" + assert fact["child_session_id"] == child_id + + +async def test_dispatch_child_aliases_resolve_to_one_canonical_child(tmp_path: Path) -> None: + """Two exact names for one child are one identity, not a contradiction. + + The progress payload names the child under both its composed identity and + its alias. Red if identity classification compares the raw provider + strings instead of the sessions they resolve to. + """ + backend = SQLiteBackend(db_path=tmp_path / "dispatch-child-aliases.db") + repo = SessionRepository(backend=backend) + try: + parent_id = await ingest_session( + parse_code( + [ + { + "type": "assistant", + "uuid": "parent-assistant", + "sessionId": "parent-native", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "dispatch-tool-1", + "name": "Task", + "input": {"description": "spawn"}, + } + ], + }, + }, + { + "type": "progress", + "uuid": "parent-progress", + "sessionId": "parent-native", + "parentToolUseID": "dispatch-tool-1", + "data": { + "type": "agent_progress", + "childSessionId": "parent-native:agent-child-native", + "agentId": "agent-child-native", + }, + }, + ], + "parent-file", + ), + backend=backend, + ) + child = parse_code(_dispatched_child_records(), "agent-child-native") + assert child.provider_session_id == "parent-native:agent-child-native" + assert child.provider_session_aliases == ["agent-child-native"] + child_id = await ingest_session(child, backend=backend) + + async with backend.connection() as conn: + [link] = await conn.execute_fetchall( + """SELECT resolved_dst_session_id, parent_tool_use_block_id, method + FROM session_links WHERE src_session_id = ?""", + (child_id,), + ) + finally: + await repo.close() + + assert link["resolved_dst_session_id"] == parent_id + assert link["parent_tool_use_block_id"] == archive_block_id( + archive_message_id(parent_id, "parent-assistant", position=0), position=0 + ) + assert link["method"] == "parent-tool-use-id" + + +async def test_dispatch_contradiction_refuses_both_candidate_children(tmp_path: Path) -> None: + """Names resolving to different sessions stay a contradiction. + + Red if canonical identity resolution degrades into accepting whichever + candidate name happens to match the child being resolved. + """ + backend = SQLiteBackend(db_path=tmp_path / "dispatch-contradiction.db") + repo = SessionRepository(backend=backend) + try: + await ingest_session( + parse_code( + [ + { + "type": "assistant", + "uuid": "parent-assistant", + "sessionId": "parent-native", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "dispatch-tool-1", + "name": "Task", + "input": {"description": "spawn"}, + } + ], + }, + }, + { + "type": "progress", + "uuid": "parent-progress-a", + "sessionId": "parent-native", + "parentToolUseID": "dispatch-tool-1", + "data": {"type": "agent_progress", "childSessionId": "parent-native:agent-child-a"}, + }, + { + "type": "progress", + "uuid": "parent-progress-b", + "sessionId": "parent-native", + "parentToolUseID": "dispatch-tool-1", + "data": {"type": "agent_progress", "childSessionId": "parent-native:agent-child-b"}, + }, + ], + "parent-file", + ), + backend=backend, + ) + child_a = await ingest_session(parse_code(_dispatched_child_records(), "agent-child-a"), backend=backend) + child_b = await ingest_session(parse_code(_dispatched_child_records(), "agent-child-b"), backend=backend) + + async with backend.connection() as conn: + links = list( + await conn.execute_fetchall( + """SELECT src_session_id, resolved_dst_session_id, parent_tool_use_block_id + FROM session_links WHERE src_session_id IN (?, ?)""", + (child_a, child_b), + ) + ) + finally: + await repo.close() + + assert {link["src_session_id"] for link in links} == {child_a, child_b} + assert all(link["parent_tool_use_block_id"] is None for link in links) From b1934d2880142c58868b40a429058d08a57fdf70 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 22:14:14 +0200 Subject: [PATCH 22/47] perf(tests): build one archive per module in the CLI snapshot and schema-generation suites test_plain_cli_snapshots took a fresh query-only lease per test (each one revalidates every artifact byte) and re-cloned plus re-materialized the cli-mixed insights for each of eight read-only consumers. test_schema_generation cloned the schema-coverage archive for each of seven tests that only read index.db. Both are now module-scoped; the per-test fixtures only set the environment the CLI reads. seeded_archive_writable has no remaining consumer and is deleted. test_completion_matrix was already module-scoped and is unchanged. --- tests/infra/corpus_fixtures.py | 12 +-- tests/infra/pathology_zoo_fixtures.py | 3 +- tests/unit/cli/test_plain_cli_snapshots.py | 89 +++++++++++++++------- tests/unit/core/test_schema_generation.py | 36 ++++++--- 4 files changed, 92 insertions(+), 48 deletions(-) diff --git a/tests/infra/corpus_fixtures.py b/tests/infra/corpus_fixtures.py index d930d985bb..5fbaccfff0 100644 --- a/tests/infra/corpus_fixtures.py +++ b/tests/infra/corpus_fixtures.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Callable, Iterator +from collections.abc import Callable from pathlib import Path import pytest @@ -45,16 +45,6 @@ def corpus_fidelity_archive(seeded_archive: SeededArchiveArtifact) -> SeededArch return seeded_archive -@pytest.fixture -def seeded_archive_writable(seeded_archive: SeededArchiveArtifact, tmp_path: Path) -> Iterator[SeededArchiveClone]: - """Private full-root clone for a mutating consumer.""" - clone = clone_seeded_archive(seeded_archive, tmp_path / "seeded-archive-clone") - try: - yield clone - finally: - clone.close() - - @pytest.fixture def named_seeded_archive( workspace_env: dict[str, Path], diff --git a/tests/infra/pathology_zoo_fixtures.py b/tests/infra/pathology_zoo_fixtures.py index cb36a433be..f7094acfdb 100644 --- a/tests/infra/pathology_zoo_fixtures.py +++ b/tests/infra/pathology_zoo_fixtures.py @@ -5,8 +5,7 @@ replay). Several test modules each need that same manifest-backed archive, some only to read it and some to mutate a private copy of it. Building it once per test session and handing out cheap clones is the same shape as -``tests.infra.corpus_fixtures.seeded_archive`` / -``seeded_archive_writable`` for the schema-coverage corpus. +``tests.infra.corpus_fixtures.seeded_archive`` for the schema-coverage corpus. """ from __future__ import annotations diff --git a/tests/unit/cli/test_plain_cli_snapshots.py b/tests/unit/cli/test_plain_cli_snapshots.py index f5eb3d8f51..262d9076eb 100644 --- a/tests/unit/cli/test_plain_cli_snapshots.py +++ b/tests/unit/cli/test_plain_cli_snapshots.py @@ -23,14 +23,22 @@ import json import re -from collections.abc import Callable +from collections.abc import Iterator from pathlib import Path from typing import cast import pytest from click.testing import CliRunner -from tests.infra.workload_artifacts import SeededArchiveClone, SeededArchiveQueryLease +from tests.infra.workload_artifacts import ( + SeededArchiveClone, + SeededArchiveQueryLease, + acquire_query_only_seeded_archive, + build_seeded_archive, + clone_seeded_archive, + named_corpus_specs, + seeded_archive_key, +) syrupy = pytest.importorskip("syrupy") @@ -73,42 +81,49 @@ def runner() -> CliRunner: return CliRunner() -@pytest.fixture -def seeded_db_env( - named_seeded_archive_ro: Callable[[str], SeededArchiveQueryLease], - monkeypatch: pytest.MonkeyPatch, -) -> Path: - """Point the CLI query verbs at a deterministic corpus DB. +@pytest.fixture(scope="module") +def query_archive_lease(request: pytest.FixtureRequest) -> SeededArchiveQueryLease: + """Pin one immutable ``cli-chatgpt`` artifact for every query snapshot here. - The named artifact is generated through the production pipeline and read - in place: every consumer of this fixture invokes read-only query verbs, - so the archive root is the immutable shared artifact rather than a - private clone of it. ``postmortem_seeded_env`` below is the mutating - counterpart and keeps the clone. + The artifact is generated through the production pipeline and read in + place: every consumer invokes read-only query verbs, so one shared lease + serves the whole module rather than one validation pass per test. """ - db_path = named_seeded_archive_ro("cli-chatgpt").path - monkeypatch.setenv("POLYLOGUE_FORCE_PLAIN", "1") - return db_path + specs = named_corpus_specs("cli-chatgpt") + artifact = build_seeded_archive(specs) + lease = acquire_query_only_seeded_archive(artifact, seeded_archive_key(specs)) + request.addfinalizer(lease.close) + return lease @pytest.fixture -def postmortem_seeded_env( - named_seeded_archive: Callable[[str], SeededArchiveClone], +def seeded_db_env( + query_archive_lease: SeededArchiveQueryLease, + workspace_env: dict[str, Path], monkeypatch: pytest.MonkeyPatch, ) -> Path: - """Seed a corpus DB and materialize the session-profile insights. + """Point the CLI query verbs at the module's deterministic corpus DB.""" + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(query_archive_lease.root)) + monkeypatch.setattr("polylogue.daemon.api_auth.load_or_mint_api_auth_token", lambda *_args, **_kwargs: None) + monkeypatch.setenv("POLYLOGUE_FORCE_PLAIN", "1") + return query_archive_lease.path + + +@pytest.fixture(scope="module") +def postmortem_archive(tmp_path_factory: pytest.TempPathFactory) -> Iterator[SeededArchiveClone]: + """Clone ``cli-mixed`` once and materialize its session-profile insights. The postmortem bundle reads durable ``session_profile`` insight records, - which the bare corpus ingest does not build. This fixture rebuilds them so - the snapshot exercises a populated bundle rather than an empty scope. + which the bare corpus ingest does not build. Rebuilding them is the + module's whole setup cost, and every consumer only runs read verbs against + the result, so it is paid once. """ import asyncio from polylogue.api import Polylogue - clone = named_seeded_archive("cli-mixed") - db_path = clone.root / "index.db" - monkeypatch.setenv("POLYLOGUE_FORCE_PLAIN", "1") + artifact = build_seeded_archive(named_corpus_specs("cli-mixed")) + clone = clone_seeded_archive(artifact, tmp_path_factory.mktemp("postmortem") / "archive") async def _rebuild() -> None: plg = Polylogue.open() @@ -117,8 +132,30 @@ async def _rebuild() -> None: finally: await plg.close() - asyncio.run(_rebuild()) - return db_path + home = tmp_path_factory.mktemp("postmortem-home") + try: + with pytest.MonkeyPatch.context() as patcher: + patcher.setenv("HOME", str(home)) + patcher.setenv("XDG_DATA_HOME", str(home / "data")) + patcher.setenv("XDG_STATE_HOME", str(home / "state")) + patcher.setenv("POLYLOGUE_SCHEMA_VALIDATION", "off") + patcher.setenv("POLYLOGUE_ARCHIVE_ROOT", str(clone.root)) + asyncio.run(_rebuild()) + yield clone + finally: + clone.close() + + +@pytest.fixture +def postmortem_seeded_env( + postmortem_archive: SeededArchiveClone, + workspace_env: dict[str, Path], + monkeypatch: pytest.MonkeyPatch, +) -> Path: + """Point the CLI at the module's insight-materialized corpus DB.""" + monkeypatch.setenv("POLYLOGUE_ARCHIVE_ROOT", str(postmortem_archive.root)) + monkeypatch.setenv("POLYLOGUE_FORCE_PLAIN", "1") + return postmortem_archive.root / "index.db" def _invoke(runner: CliRunner, args: list[str]) -> str: diff --git a/tests/unit/core/test_schema_generation.py b/tests/unit/core/test_schema_generation.py index 8e6137a13a..c2e3fbe4a1 100644 --- a/tests/unit/core/test_schema_generation.py +++ b/tests/unit/core/test_schema_generation.py @@ -33,11 +33,29 @@ ) from polylogue.schemas.packages import SchemaElementManifest, SchemaPackageCatalog, SchemaVersionPackage from tests.infra.schema_access import schema_properties, schema_property, schema_values -from tests.infra.workload_artifacts import SeededArchiveClone +from tests.infra.workload_artifacts import SeededArchiveArtifact, clone_seeded_archive pytest_plugins = ("tests.infra.corpus_fixtures",) +@pytest.fixture(scope="module") +def schema_sample_db( + seeded_archive: SeededArchiveArtifact, + tmp_path_factory: pytest.TempPathFactory, +) -> Generator[Path, None, None]: + """One schema-coverage clone shared by every sample-reading test here. + + The consumers below only read ``index.db``, so one clone serves the whole + module. A clone rather than the sealed artifact because the sampling + readers open the database read-write. + """ + clone = clone_seeded_archive(seeded_archive, tmp_path_factory.mktemp("schema-generation") / "archive") + try: + yield clone.root / "index.db" + finally: + clone.close() + + class TestProviderSchemaGeneration: """Provider-level schema generation entrypoints.""" @@ -48,8 +66,8 @@ def test_known_providers(self) -> None: @pytest.mark.slow @pytest.mark.parametrize("provider", ["chatgpt", "claude-code", "codex"]) - def test_generate_schema_from_db(self, seeded_archive_writable: SeededArchiveClone, provider: str) -> None: - result = generate_provider_schema(provider, db_path=seeded_archive_writable.root / "index.db", max_samples=100) + def test_generate_schema_from_db(self, schema_sample_db: Path, provider: str) -> None: + result = generate_provider_schema(provider, db_path=schema_sample_db, max_samples=100) if result.sample_count > 0: assert result.success, f"Failed: {result.error}" assert result.schema is not None @@ -70,11 +88,11 @@ def test_result_dataclass(self) -> None: assert not failure.success -def test_generation_records_aggregate_phase_receipt(seeded_archive_writable: SeededArchiveClone) -> None: +def test_generation_records_aggregate_phase_receipt(schema_sample_db: Path) -> None: events: list[JSONDocument] = [] result = generate_provider_schema( "chatgpt", - db_path=seeded_archive_writable.root / "index.db", + db_path=schema_sample_db, max_samples=10, progress_callback=lambda _phase, payload: events.append(payload), ) @@ -128,12 +146,12 @@ def package(version: str, *, scopes: int, samples: int, first_seen: str) -> Sche class TestLoadSamples: """Database-backed sample loading behavior.""" - def test_load_limited_samples(self, seeded_archive_writable: SeededArchiveClone) -> None: - samples = load_samples_from_db("chatgpt", db_path=seeded_archive_writable.root / "index.db", max_samples=10) + def test_load_limited_samples(self, schema_sample_db: Path) -> None: + samples = load_samples_from_db("chatgpt", db_path=schema_sample_db, max_samples=10) assert len(samples) <= 10 - def test_load_nonexistent_provider(self, seeded_archive_writable: SeededArchiveClone) -> None: - assert load_samples_from_db("nonexistent-provider", db_path=seeded_archive_writable.root / "index.db") == [] + def test_load_nonexistent_provider(self, schema_sample_db: Path) -> None: + assert load_samples_from_db("nonexistent-provider", db_path=schema_sample_db) == [] def test_load_limited_document_samples_stops_without_full_materialization( self, From c26cacbda6c39b9c2aca58c295710619fe2b4fda Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 01:57:09 +0200 Subject: [PATCH 23/47] test(cli): reflect absent optional status relations --- .../unit/cli/__snapshots__/test_plain_cli_snapshots.ambr | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr b/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr index 786baec00b..bee143a9d0 100644 --- a/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr +++ b/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr @@ -1065,9 +1065,7 @@ "blocks": 38, "session_profiles": 0, "session_work_events": 0, - "session_phases": 0, - "threads": 2, - "thread_sessions": 2 + "session_phases": 0 }, "table_count_precision": { "sessions": "exact", @@ -1075,9 +1073,7 @@ "blocks": "exact", "session_profiles": "exact", "session_work_events": "exact", - "session_phases": "exact", - "threads": "exact", - "thread_sessions": "exact" + "session_phases": "exact" } }, "embeddings": { From 54a9c4eac4d30492ae70f4b039ff2b8ba698f9ac Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 14:43:47 +0200 Subject: [PATCH 24/47] fix: preserve exact FTS staleness through search --- .../pipeline/services/ingest_batch/_core.py | 8 + polylogue/storage/fts/freshness.py | 14 ++ polylogue/storage/fts/fts_lifecycle.py | 32 +++- polylogue/storage/fts/session_repair.py | 24 ++- .../storage/sqlite/queries/sessions_search.py | 158 ++++++++++-------- 5 files changed, 159 insertions(+), 77 deletions(-) diff --git a/polylogue/pipeline/services/ingest_batch/_core.py b/polylogue/pipeline/services/ingest_batch/_core.py index f0835836b1..4b13f2ef34 100644 --- a/polylogue/pipeline/services/ingest_batch/_core.py +++ b/polylogue/pipeline/services/ingest_batch/_core.py @@ -1386,6 +1386,14 @@ def _drain_ready_session_entries( source_conn: sqlite3.Connection | None = None, ) -> int: _delete_stale_sessions_for_raw_entries(conn, ready_entries) + from polylogue.storage.fts.freshness import message_fts_recorded_exact_stale_sync + + if message_fts_recorded_exact_stale_sync(conn): + # An exact stale verdict is archive-wide negative evidence. Keep the + # materialized sessions in the repair queue even when cleanup leaves + # equal row counts and the session-local probe has no missing row. + for _raw_id, cdata in ready_entries: + summary.fts_repair_session_ids.append(cdata.session_id) written_count = 0 # One signature cache per drained batch memoizes each session's own composed # signatures so a parent with K fork-children is computed once, not K times diff --git a/polylogue/storage/fts/freshness.py b/polylogue/storage/fts/freshness.py index e22f8172bd..cd1cd816e1 100644 --- a/polylogue/storage/fts/freshness.py +++ b/polylogue/storage/fts/freshness.py @@ -587,6 +587,12 @@ def message_fts_recorded_readiness_sync(conn: sqlite3.Connection) -> dict[str, i return None +def message_fts_recorded_exact_stale_sync(conn: sqlite3.Connection) -> bool: + """Return whether the ledger contains an exact negative verdict.""" + record = _message_fts_record_sync(conn) + return record is not None and str(record["state"]) == STALE and str(record.get("verification_kind")) == EXACT + + async def message_fts_recorded_readiness_async(conn: aiosqlite.Connection) -> dict[str, int | bool] | None: """Async counterpart to :func:`message_fts_recorded_readiness_sync`.""" record = await _message_fts_record_async(conn) @@ -610,6 +616,12 @@ async def message_fts_recorded_readiness_async(conn: aiosqlite.Connection) -> di return None +async def message_fts_recorded_exact_stale_async(conn: aiosqlite.Connection) -> bool: + """Return whether the ledger contains an exact negative verdict.""" + record = await _message_fts_record_async(conn) + return record is not None and str(record["state"]) == STALE and str(record.get("verification_kind")) == EXACT + + def message_fts_recorded_ready_trusted_sync(conn: sqlite3.Connection) -> bool: record = _message_fts_record_sync(conn) if record is None: @@ -648,6 +660,8 @@ async def message_fts_recorded_ready_trusted_async(conn: aiosqlite.Connection) - "message_fts_recorded_ready_trusted_sync", "message_fts_recorded_readiness_async", "message_fts_recorded_readiness_sync", + "message_fts_recorded_exact_stale_async", + "message_fts_recorded_exact_stale_sync", "message_fts_recorded_state_async", "message_fts_recorded_state_sync", "record_fts_invariant_snapshot_sync", diff --git a/polylogue/storage/fts/fts_lifecycle.py b/polylogue/storage/fts/fts_lifecycle.py index f2a0d42625..56d2352bac 100644 --- a/polylogue/storage/fts/fts_lifecycle.py +++ b/polylogue/storage/fts/fts_lifecycle.py @@ -777,11 +777,24 @@ def message_fts_readiness_sync( def message_fts_search_readiness_sync(conn: sqlite3.Connection) -> dict[str, int | bool]: """Return retrieval readiness measured for the relation used by search.""" - from polylogue.storage.fts.freshness import message_fts_recorded_readiness_sync + from polylogue.storage.fts.freshness import ( + message_fts_recorded_exact_stale_sync, + message_fts_recorded_readiness_sync, + ) recorded_readiness = message_fts_recorded_readiness_sync(conn) if recorded_readiness is not None: return recorded_readiness + if message_fts_recorded_exact_stale_sync(conn): + status = fts_index_status_sync(conn) + return { + "exists": bool(status.get("exists", False)), + "indexed_rows": _status_int(status, "count"), + "total_rows": _row_int(conn.execute(FTS_INDEXABLE_MESSAGE_COUNT_SQL).fetchone(), 0), + "ready": False, + "triggers_present": bool(status.get("exists", False)) + and _triggers_present_sync(conn, _message_trigger_names_for_sync(conn)), + } readiness = message_fts_readiness_sync(conn, verify_total_rows=True) return readiness @@ -823,11 +836,26 @@ async def message_fts_readiness_async( async def message_fts_search_readiness_async(conn: aiosqlite.Connection) -> dict[str, int | bool]: """Async retrieval readiness measured for the relation used by search.""" - from polylogue.storage.fts.freshness import message_fts_recorded_readiness_async + from polylogue.storage.fts.freshness import ( + message_fts_recorded_exact_stale_async, + message_fts_recorded_readiness_async, + ) recorded_readiness = await message_fts_recorded_readiness_async(conn) if recorded_readiness is not None: return recorded_readiness + if await message_fts_recorded_exact_stale_async(conn): + status = await fts_index_status_async(conn) + exists = bool(status.get("exists", False)) + row = await (await conn.execute(FTS_INDEXABLE_MESSAGE_COUNT_SQL)).fetchone() + return { + "exists": exists, + "indexed_rows": _status_int(status, "count"), + "total_rows": _row_int(row, 0), + "ready": False, + "triggers_present": exists + and await _triggers_present_async(conn, await _message_trigger_names_for_async(conn)), + } readiness = await message_fts_readiness_async(conn, verify_total_rows=True) return readiness diff --git a/polylogue/storage/fts/session_repair.py b/polylogue/storage/fts/session_repair.py index 986462ade0..8cddcb6d42 100644 --- a/polylogue/storage/fts/session_repair.py +++ b/polylogue/storage/fts/session_repair.py @@ -5,6 +5,7 @@ import sqlite3 from typing import Any, cast +from polylogue.storage.fts.sql import FTS_MESSAGES_IDENTITY_RECIPE_ID from polylogue.storage.introspection import table_exists as _table_exists @@ -18,7 +19,7 @@ def _row_int(row: sqlite3.Row | tuple[object, ...] | None, key: int | str) -> in def session_fts_needs_repair_sync(conn: sqlite3.Connection, session_id: str) -> bool: - """Return whether one session has missing message FTS rows.""" + """Return whether one session has missing or identity-drifted FTS rows.""" if not session_id: return False if not _table_exists(conn, "messages_fts_docsize"): @@ -37,7 +38,26 @@ def session_fts_needs_repair_sync(conn: sqlite3.Connection, session_id: str) -> ).fetchone(), 0, ) - return missing_blocks > 0 + if missing_blocks > 0: + return True + if not _table_exists(conn, "messages_fts_identity"): + return False + mismatch = _row_int( + conn.execute( + """ + SELECT COUNT(*) + FROM messages_fts_docsize AS d + JOIN blocks AS b ON b.rowid = d.id AND b.session_id = ? AND b.search_text != '' + JOIN messages_fts_identity AS i ON i.rowid = d.id + WHERE i.block_id != b.block_id + OR i.source_hash IS NOT b.content_hash + OR i.recipe_id != ? + """, + (session_id, FTS_MESSAGES_IDENTITY_RECIPE_ID), + ).fetchone(), + 0, + ) + return mismatch > 0 def repair_session_fts_if_needed_sync(conn: sqlite3.Connection, session_id: str) -> bool: diff --git a/polylogue/storage/sqlite/queries/sessions_search.py b/polylogue/storage/sqlite/queries/sessions_search.py index 6b96e41137..9a84ba781d 100644 --- a/polylogue/storage/sqlite/queries/sessions_search.py +++ b/polylogue/storage/sqlite/queries/sessions_search.py @@ -4,7 +4,10 @@ import aiosqlite +from polylogue.storage.fts.fts_lifecycle import check_fts_readiness, message_fts_search_readiness_async +from polylogue.storage.search import build_ranked_action_search_query, build_ranked_session_search_query from polylogue.storage.search.models import SessionSearchEvidenceRow, SessionSearchResult +from polylogue.storage.search.query_support import extract_match_terms async def search_session_hits( @@ -13,28 +16,31 @@ async def search_session_hits( limit: int = 100, origins: list[str] | None = None, ) -> SessionSearchResult: - from polylogue.storage.fts.fts_lifecycle import check_fts_readiness, message_fts_search_readiness_async - # Search must not silently serve stale FTS results. Status/reporting # paths may use bounded structural probes, but retrieval is a hard # correctness boundary. - readiness = await message_fts_search_readiness_async(conn) - check_fts_readiness(readiness) - - from polylogue.storage.search import build_ranked_session_search_query - - query_spec = build_ranked_session_search_query( - query=query, - limit=limit, - scope_names=origins, - ) - if query_spec is None: - return SessionSearchResult(hits=[]) + owns_snapshot = not conn.in_transaction + if owns_snapshot: + await conn.execute("BEGIN") + try: + readiness = await message_fts_search_readiness_async(conn) + check_fts_readiness(readiness) + + query_spec = build_ranked_session_search_query( + query=query, + limit=limit, + scope_names=origins, + ) + if query_spec is None: + return SessionSearchResult(hits=[]) - sql, params = query_spec.sql, query_spec.params - cursor = await conn.execute(sql, params) - rows = await cursor.fetchall() - return SessionSearchResult.from_ids([str(row["session_id"]) for row in rows]) + sql, params = query_spec.sql, query_spec.params + cursor = await conn.execute(sql, params) + rows = await cursor.fetchall() + return SessionSearchResult.from_ids([str(row["session_id"]) for row in rows]) + finally: + if owns_snapshot: + await conn.rollback() async def search_session_evidence_hits( @@ -44,46 +50,48 @@ async def search_session_evidence_hits( origins: list[str] | None = None, since: str | None = None, ) -> list[SessionSearchEvidenceRow]: - from polylogue.storage.fts.fts_lifecycle import check_fts_readiness, message_fts_search_readiness_async - from polylogue.storage.search import build_ranked_session_search_query - # See search_session_hits: retrieval is allowed only against an # exactly fresh message FTS surface. - readiness = await message_fts_search_readiness_async(conn) - check_fts_readiness(readiness) - - query_spec = build_ranked_session_search_query( - query=query, - limit=limit, - scope_names=origins, - since=since, - include_snippet=True, - ) - if query_spec is None: - return [] - - cursor = await conn.execute(query_spec.sql, query_spec.params) - rows = await cursor.fetchall() - from polylogue.storage.search.query_support import extract_match_terms - - matched_terms = extract_match_terms(query) - return [ - SessionSearchEvidenceRow( - session_id=str(row["session_id"]), - rank=rank, - score=float(row["relevance"]) if row["relevance"] is not None else None, - message_id=str(row["message_id"]) if row["message_id"] is not None else None, - snippet=str(row["snippet"] or row["fallback_text"] or ""), - match_surface="message", - retrieval_lane="dialogue", - matched_terms=matched_terms, - score_components=({"bm25_raw": float(row["relevance"])} if row["relevance"] is not None else {}), - score_kind="bm25" if row["relevance"] is not None else None, - lane_rank=rank, - raw_score=float(row["relevance"]) if row["relevance"] is not None else None, + owns_snapshot = not conn.in_transaction + if owns_snapshot: + await conn.execute("BEGIN") + try: + readiness = await message_fts_search_readiness_async(conn) + check_fts_readiness(readiness) + + query_spec = build_ranked_session_search_query( + query=query, + limit=limit, + scope_names=origins, + since=since, + include_snippet=True, ) - for rank, row in enumerate(rows, start=1) - ] + if query_spec is None: + return [] + + cursor = await conn.execute(query_spec.sql, query_spec.params) + rows = await cursor.fetchall() + matched_terms = extract_match_terms(query) + return [ + SessionSearchEvidenceRow( + session_id=str(row["session_id"]), + rank=rank, + score=float(row["relevance"]) if row["relevance"] is not None else None, + message_id=str(row["message_id"]) if row["message_id"] is not None else None, + snippet=str(row["snippet"] or row["fallback_text"] or ""), + match_surface="message", + retrieval_lane="dialogue", + matched_terms=matched_terms, + score_components=({"bm25_raw": float(row["relevance"])} if row["relevance"] is not None else {}), + score_kind="bm25" if row["relevance"] is not None else None, + lane_rank=rank, + raw_score=float(row["relevance"]) if row["relevance"] is not None else None, + ) + for rank, row in enumerate(rows, start=1) + ] + finally: + if owns_snapshot: + await conn.rollback() async def search_sessions( @@ -101,24 +109,28 @@ async def search_action_session_hits( limit: int = 100, origins: list[str] | None = None, ) -> SessionSearchResult: - from polylogue.storage.fts.fts_lifecycle import check_fts_readiness, message_fts_search_readiness_async - from polylogue.storage.search import build_ranked_action_search_query - - readiness = await message_fts_search_readiness_async(conn) - check_fts_readiness(readiness) - - query_spec = build_ranked_action_search_query( - query=query, - limit=limit, - scope_names=origins, - ) - if query_spec is None: - return SessionSearchResult(hits=[]) - - sql, params = query_spec.sql, query_spec.params - cursor = await conn.execute(sql, params) - rows = await cursor.fetchall() - return SessionSearchResult.from_ids([str(row["session_id"]) for row in rows]) + owns_snapshot = not conn.in_transaction + if owns_snapshot: + await conn.execute("BEGIN") + try: + readiness = await message_fts_search_readiness_async(conn) + check_fts_readiness(readiness) + + query_spec = build_ranked_action_search_query( + query=query, + limit=limit, + scope_names=origins, + ) + if query_spec is None: + return SessionSearchResult(hits=[]) + + sql, params = query_spec.sql, query_spec.params + cursor = await conn.execute(sql, params) + rows = await cursor.fetchall() + return SessionSearchResult.from_ids([str(row["session_id"]) for row in rows]) + finally: + if owns_snapshot: + await conn.rollback() async def search_action_sessions( From 9be123e6aba0890348fa079dfcd32fbcee37a293 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 16:32:47 +0200 Subject: [PATCH 25/47] test(fts): seed exact freshness before scoped repair --- tests/unit/pipeline/test_ingest_batch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/pipeline/test_ingest_batch.py b/tests/unit/pipeline/test_ingest_batch.py index dcff5c1d68..8bcc01b7f6 100644 --- a/tests/unit/pipeline/test_ingest_batch.py +++ b/tests/unit/pipeline/test_ingest_batch.py @@ -68,6 +68,8 @@ from polylogue.storage.blob_publication import ArchiveBlobPublisher from polylogue.storage.blob_store import BlobStore from polylogue.storage.derived.session.refresh import SessionInsightRefreshChunkObservation +from polylogue.storage.fts.freshness import record_fts_invariant_snapshot_sync +from polylogue.storage.fts.fts_lifecycle import fts_invariant_snapshot_sync from polylogue.storage.raw.models import RawSessionStateUpdate from polylogue.storage.raw_failure_lifecycle import read_raw_failure_lifecycle from polylogue.storage.repository import SessionRepository @@ -3058,6 +3060,7 @@ def test_process_ingest_batch_sync_commits_targeted_fts_repair_and_invalidates_s ) with open_connection(db_path) as conn: + record_fts_invariant_snapshot_sync(conn, fts_invariant_snapshot_sync(conn)) conn.commit() first_result = search_messages(needle, archive_root=archive_root, db_path=db_path, limit=10) From 577242f6f14690c5b64ad476366581343a57ccdc Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 02:01:42 +0200 Subject: [PATCH 26/47] test(fts): preserve exact stale identity evidence --- .../pipeline/test_ingest_batch_fts_repair.py | 99 ++++++++++ .../unit/storage/test_fts_identity_ledger.py | 149 +++++++++++++++ .../storage/test_search_snapshot_ownership.py | 176 ++++++++++++++++++ 3 files changed, 424 insertions(+) create mode 100644 tests/unit/storage/test_search_snapshot_ownership.py diff --git a/tests/unit/pipeline/test_ingest_batch_fts_repair.py b/tests/unit/pipeline/test_ingest_batch_fts_repair.py index fea86b81d7..d2a7c83c61 100644 --- a/tests/unit/pipeline/test_ingest_batch_fts_repair.py +++ b/tests/unit/pipeline/test_ingest_batch_fts_repair.py @@ -9,6 +9,12 @@ import polylogue.pipeline.services.ingest_batch._core as ingest_batch_core from polylogue.pipeline.services.ingest_batch import _process_ingest_batch_sync from polylogue.pipeline.services.ingest_worker import IngestRecordResult +from polylogue.storage.fts.freshness import ( + message_fts_recorded_exact_stale_sync, + record_fts_invariant_snapshot_sync, +) +from polylogue.storage.fts.fts_lifecycle import fts_invariant_snapshot_sync +from polylogue.storage.fts.session_repair import session_fts_needs_repair_sync from polylogue.storage.runtime import RawSessionRecord from polylogue.storage.sqlite.connection import open_connection from tests.unit.pipeline.test_ingest_batch import _message_tuple, _session_data @@ -112,3 +118,96 @@ def fake_ingest_record( ).fetchone()[0] assert message_fts_count == 1 + + +def test_process_ingest_batch_keeps_repair_when_recorded_state_is_exact_stale( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An exact stale verdict keeps materialized sessions queued for repair. + + The session-local probe (``session_fts_needs_repair_sync``) only sees its + own session. When the archive carries an archive-wide exact stale verdict, + a drained session whose own rows happen to balance must still be scheduled + -- otherwise cleanup silently drops the repair that would clear the + verdict and the surface stays unsearchable. + + Mutation that fails this: remove the ``message_fts_recorded_exact_stale_sync`` + branch from ``_drain_ready_session_entries``. Nothing is missing for this + session, so ``fts_repair_session_ids`` comes back empty. + """ + db_path = tmp_path / "index.db" + archive_root = tmp_path / "archive" + blob_root = tmp_path / "blob" + source_path = tmp_path / "raw.jsonl" + source_path.write_text("{}", encoding="utf-8") + raw_record = RawSessionRecord( + raw_id="raw-exact-stale-fts", + source_name="codex", + source_path=str(source_path), + blob_size=source_path.stat().st_size, + acquired_at="2026-04-02T00:00:00Z", + ) + session_id = "codex-session:exact-stale-fts" + session = _session_data( + session_id, + content_hash="hash-exact-stale-fts", + message_tuples=[ + _message_tuple( + "msg-exact-stale-fts", + session_id, + role="user", + text="content whose own FTS rows are complete", + content_hash="hash-exact-stale-message", + sort_key=0.0, + ) + ], + ) + + with open_connection(db_path) as conn: + changed, _counts = _write_session(conn, session) + assert changed is True + from polylogue.storage.fts.fts_lifecycle import repair_fts_index_sync + + repair_fts_index_sync(conn, [session_id]) + conn.commit() + + # This session is internally complete: the session-local probe alone + # would drop it from the repair queue. + assert session_fts_needs_repair_sync(conn, session_id) is False + + # Record an archive-wide EXACT stale verdict without touching this + # session's rows, the shape a deferred repair elsewhere leaves behind. + snapshot = fts_invariant_snapshot_sync(conn) + record_fts_invariant_snapshot_sync(conn, snapshot) + conn.execute( + "UPDATE fts_freshness_state SET state = 'stale', identity_mismatch_rows = 1 WHERE surface = 'messages_fts'" + ) + conn.commit() + assert message_fts_recorded_exact_stale_sync(conn) is True + + def fake_ingest_record( + record: RawSessionRecord, + archive_root_str: str, + validation_mode: str, + measure_ingest_result_size: bool, + *, + blob_root_str: str | None, + ) -> IngestRecordResult: + del archive_root_str, validation_mode, measure_ingest_result_size, blob_root_str + assert record.raw_id == raw_record.raw_id + return IngestRecordResult(raw_id=record.raw_id, sessions=[session]) + + monkeypatch.setattr(ingest_batch_core, "ingest_record", fake_ingest_record) + + summary = _process_ingest_batch_sync( + [raw_record], + db_path=db_path, + archive_root_str=str(archive_root), + blob_root_str=str(blob_root), + validation_mode="off", + ingest_workers=1, + measure_ingest_result_size=False, + ) + + assert session_id in summary.fts_repair_session_ids diff --git a/tests/unit/storage/test_fts_identity_ledger.py b/tests/unit/storage/test_fts_identity_ledger.py index fddd9b4f04..0439550b0d 100644 --- a/tests/unit/storage/test_fts_identity_ledger.py +++ b/tests/unit/storage/test_fts_identity_ledger.py @@ -15,9 +15,12 @@ from __future__ import annotations +import asyncio import sqlite3 +from pathlib import Path from typing import TYPE_CHECKING +import aiosqlite import pytest from polylogue.storage.fts.freshness import ( @@ -30,11 +33,16 @@ ) from polylogue.storage.fts.fts_lifecycle import ( fts_invariant_snapshot_sync, + message_fts_readiness_sync, + message_fts_search_readiness_async, + message_fts_search_readiness_sync, restore_fts_triggers_sync, ) +from polylogue.storage.fts.session_repair import session_fts_needs_repair_sync from polylogue.storage.fts.sql import FTS_MESSAGES_IDENTITY_RECIPE_ID, message_identity_mismatch_sql from polylogue.storage.sqlite.archive_tiers.index import INDEX_SCHEMA_VERSION from polylogue.storage.sqlite.archive_tiers.ops_write import list_fts_drift_samples, record_fts_drift_sample +from polylogue.storage.sqlite.connection import open_connection from tests.infra.identity import archive_message_id if TYPE_CHECKING: @@ -406,6 +414,147 @@ def test_missing_identity_row_is_not_counted_as_mismatch(self, test_conn: sqlite assert _identity_mismatch_count(test_conn) == 0 +class TestExactStaleSurvivesCountOnlyFallback: + """An exact negative verdict is authority; equal counts must not erase it. + + A recorded ``stale``/``exact`` row is deliberately NOT returned as a + trusted readiness verdict (a stale row is a cache miss for the *positive* + case, polylogue-g0s6k.2), so the search path falls through to a live + measurement. That measurement is count-only -- ``ready = exists and + triggers_present and indexed_rows == total_rows`` -- and an identity + substitution keeps both counts equal by construction. Without an explicit + exact-stale branch the fallback therefore *upgrades* a proven-invalid + surface back to ready and search serves stale postings. + """ + + @staticmethod + def _identity_substituted_exact_stale(conn: sqlite3.Connection) -> None: + """Leave the archive with equal counts and a recorded exact-stale row.""" + restore_fts_triggers_sync(conn) + block_id = _seed_block( + conn, + native_session_id="conv-exact-stale-search", + native_message_id="msg-exact-stale-search", + text="genuine current block", + content_hash=b"e" * 32, + ) + rowid = _block_rowid(conn, block_id) + conn.execute( + "UPDATE messages_fts_identity SET block_id = 'stale:ghost:0' WHERE rowid = ?", + (rowid,), + ) + snapshot = fts_invariant_snapshot_sync(conn) + assert snapshot.messages.identity_mismatch_rows == 1 + assert snapshot.messages.source_rows == snapshot.messages.indexed_rows + record_fts_invariant_snapshot_sync(conn, snapshot) + conn.commit() + + def test_count_only_fallback_would_call_this_surface_ready(self, test_conn: sqlite3.Connection) -> None: + """Pin the hazard itself, so the next two tests cannot pass vacuously. + + If this assertion ever flips to ``False`` the count-only measurement + grew its own identity check and the exact-stale branch in + ``message_fts_search_readiness_*`` is no longer load-bearing -- delete + it rather than leaving two checks disagreeing. + """ + self._identity_substituted_exact_stale(test_conn) + + assert message_fts_readiness_sync(test_conn, verify_total_rows=True)["ready"] is True + + def test_search_readiness_sync_keeps_exact_stale_verdict(self, test_conn: sqlite3.Connection) -> None: + """Mutation that fails this: drop the ``message_fts_recorded_exact_stale_sync`` + branch from ``message_fts_search_readiness_sync`` -- the count-only + fallback then reports ``ready=True`` (pinned by the test above). + """ + self._identity_substituted_exact_stale(test_conn) + + readiness = message_fts_search_readiness_sync(test_conn) + + assert readiness["ready"] is False + assert readiness["indexed_rows"] == readiness["total_rows"], ( + "the regression only bites while counts agree -- if they diverge " + "here the fixture stopped reproducing the identity-substitution shape" + ) + + def test_search_readiness_async_keeps_exact_stale_verdict(self, test_db: Path) -> None: + """The async retrieval path must refuse on the same evidence as sync.""" + with open_connection(test_db) as conn: + self._identity_substituted_exact_stale(conn) + + async def _measure() -> dict[str, int | bool]: + async with aiosqlite.connect(test_db) as conn: + conn.row_factory = aiosqlite.Row + return await message_fts_search_readiness_async(conn) + + readiness = asyncio.run(_measure()) + + assert readiness["ready"] is False + assert readiness["indexed_rows"] == readiness["total_rows"] + + +class TestSessionRepairSeesIdentityDrift: + """Scoped repair must key on identity, not only on missing rows. + + ``session_fts_needs_repair_sync`` gates the targeted repair the ingest + path schedules. A rowid that silently rebound to a different block leaves + every ``messages_fts_docsize`` row present, so the missing-blocks probe + alone reports the session healthy and the drift is never repaired. + """ + + def test_identity_mismatch_alone_requires_repair(self, test_conn: sqlite3.Connection) -> None: + """Mutation that fails this: restore the bare ``return missing_blocks > 0`` + in ``session_fts_needs_repair_sync``. + """ + restore_fts_triggers_sync(test_conn) + block_id = _seed_block( + test_conn, + native_session_id="conv-session-repair-identity", + native_message_id="msg-session-repair-identity", + text="block whose ledger row will drift", + content_hash=b"d" * 32, + ) + session_id = str( + test_conn.execute("SELECT session_id FROM blocks WHERE block_id = ?", (block_id,)).fetchone()[0] + ) + rowid = _block_rowid(test_conn, block_id) + assert session_fts_needs_repair_sync(test_conn, session_id) is False, ( + "a consistent session must not be queued for repair -- otherwise " + "this test would pass without the identity arm" + ) + + test_conn.execute( + "UPDATE messages_fts_identity SET block_id = 'stale:ghost:0' WHERE rowid = ?", + (rowid,), + ) + + docsize_present = test_conn.execute("SELECT 1 FROM messages_fts_docsize WHERE id = ?", (rowid,)).fetchone() + assert docsize_present is not None, "no row is missing -- only its identity drifted" + assert session_fts_needs_repair_sync(test_conn, session_id) is True + + def test_stale_source_hash_requires_repair(self, test_conn: sqlite3.Connection) -> None: + """The ledger's ``source_hash`` arm gates repair too, not just block_id.""" + restore_fts_triggers_sync(test_conn) + block_id = _seed_block( + test_conn, + native_session_id="conv-session-repair-hash", + native_message_id="msg-session-repair-hash", + text="block whose ledgered hash will drift", + content_hash=b"c" * 32, + ) + session_id = str( + test_conn.execute("SELECT session_id FROM blocks WHERE block_id = ?", (block_id,)).fetchone()[0] + ) + rowid = _block_rowid(test_conn, block_id) + assert session_fts_needs_repair_sync(test_conn, session_id) is False + + test_conn.execute( + "UPDATE messages_fts_identity SET source_hash = ? WHERE rowid = ?", + (b"0" * 32, rowid), + ) + + assert session_fts_needs_repair_sync(test_conn, session_id) is True + + class TestIdentityMismatchGatesReadiness: """Wired into the same readiness contract missing_rows/excess_rows use.""" diff --git a/tests/unit/storage/test_search_snapshot_ownership.py b/tests/unit/storage/test_search_snapshot_ownership.py new file mode 100644 index 0000000000..6f8764559b --- /dev/null +++ b/tests/unit/storage/test_search_snapshot_ownership.py @@ -0,0 +1,176 @@ +"""Retrieval reads readiness and postings under one snapshot (polylogue-5guoo). + +``message_fts_search_readiness_*`` may fall back to a live measurement that +compares two independently-counted relations (``messages_fts_docsize`` against +the indexable ``blocks`` denominator). On an autocommit connection those two +counts are two separate read snapshots, so an unrelated commit landing between +them makes the counts disagree and ``check_fts_readiness`` refuses a perfectly +healthy archive. The daemon commits continuously, so this is the ordinary live +condition, not an exotic race. + +The search entry points therefore open their own deferred read transaction +when -- and only when -- the caller is not already inside one, and release it +with ``rollback`` so retrieval never becomes a writer. +""" + +from __future__ import annotations + +import itertools +import sqlite3 +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Any + +import aiosqlite +import pytest + +import polylogue.storage.fts.fts_lifecycle as fts_lifecycle +from polylogue.storage.fts.freshness import record_fts_invariant_snapshot_sync +from polylogue.storage.fts.fts_lifecycle import fts_invariant_snapshot_sync +from polylogue.storage.sqlite.connection import open_connection +from polylogue.storage.sqlite.queries.sessions_search import ( + search_action_session_hits, + search_session_evidence_hits, + search_session_hits, +) +from tests.infra.identity import archive_message_id + +_ORIGIN = "unknown-export" +_TERM = "snapshot" + +SearchEntryPoint = Callable[..., Awaitable[Any]] + +# Every retrieval entry point in sessions_search that measures readiness and +# then matches; each must take the same snapshot decision. +_ENTRY_POINTS: tuple[SearchEntryPoint, ...] = ( + search_session_hits, + search_session_evidence_hits, + search_action_session_hits, +) + + +def _seed_session(conn: sqlite3.Connection, native_session_id: str, text: str) -> None: + """Insert one minimal searchable session/message/block.""" + session_id = f"{_ORIGIN}:{native_session_id}" + content_hash = b"e" * 32 + message_id = archive_message_id(session_id, "m0", position=0) + conn.execute( + "INSERT OR IGNORE INTO sessions (native_id, origin, title, content_hash) VALUES (?, ?, ?, ?)", + (native_session_id, _ORIGIN, "snapshot ownership", content_hash), + ) + conn.execute( + """ + INSERT INTO messages (session_id, native_id, position, role, message_type, content_hash) + VALUES (?, 'm0', 0, 'user', 'message', ?) + """, + (session_id, content_hash), + ) + conn.execute( + """ + INSERT INTO blocks (message_id, session_id, position, block_type, text, content_hash) + VALUES (?, ?, 0, 'text', ?, ?) + """, + (message_id, session_id, text, content_hash), + ) + + +@pytest.fixture +def searchable_db(tmp_path: Path) -> Path: + """An archive whose message FTS is exactly fresh and answers ``_TERM``.""" + db_path = tmp_path / "index.db" + with open_connection(db_path) as conn: + _seed_session(conn, "conv-snapshot", f"searchable {_TERM} content") + record_fts_invariant_snapshot_sync(conn, fts_invariant_snapshot_sync(conn)) + conn.commit() + return db_path + + +@pytest.mark.parametrize("entry_point", _ENTRY_POINTS, ids=lambda fn: fn.__name__) +async def test_search_releases_the_snapshot_it_opened(searchable_db: Path, entry_point: SearchEntryPoint) -> None: + """An autocommit caller must be handed back an autocommit connection. + + Mutation that fails this: drop the ``finally``/``rollback`` arm. The + connection then stays inside the deferred transaction search opened, + pinning a WAL read snapshot open for the rest of the caller's life. + """ + async with aiosqlite.connect(searchable_db) as conn: + conn.row_factory = aiosqlite.Row + assert conn.in_transaction is False + + await entry_point(conn, _TERM, limit=5) + + assert conn.in_transaction is False + + +@pytest.mark.parametrize("entry_point", _ENTRY_POINTS, ids=lambda fn: fn.__name__) +async def test_search_does_not_discard_a_caller_transaction(searchable_db: Path, entry_point: SearchEntryPoint) -> None: + """Search must not roll back writes it does not own. + + Mutation that fails this: hardcode ``owns_snapshot = True``. The + ``finally`` arm then rolls back the caller's still-pending INSERT and both + assertions below fail -- retrieval would silently destroy an in-flight + ingest transaction that happened to run a search. + """ + async with aiosqlite.connect(searchable_db) as conn: + conn.row_factory = aiosqlite.Row + await conn.execute("BEGIN") + await conn.execute( + "INSERT INTO sessions (native_id, origin, title, content_hash) VALUES (?, ?, ?, ?)", + ("pending-caller-write", _ORIGIN, "uncommitted", b"p" * 32), + ) + assert conn.in_transaction is True + + await entry_point(conn, _TERM, limit=5) + + assert conn.in_transaction is True, "search took ownership of a transaction it did not open" + row = await ( + await conn.execute("SELECT COUNT(*) FROM sessions WHERE native_id = 'pending-caller-write'") + ).fetchone() + assert row is not None + assert int(row[0]) == 1, "search rolled back the caller's pending write" + + +@pytest.mark.parametrize("entry_point", _ENTRY_POINTS, ids=lambda fn: fn.__name__) +async def test_commit_between_readiness_probes_does_not_refuse( + searchable_db: Path, + entry_point: SearchEntryPoint, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A commit landing mid-measurement must not fake an incomplete index. + + The count-only fallback runs only when no trusted freshness record exists, + so the record is removed first. ``fts_index_status_async`` is then wrapped + to commit a new indexable block from a *separate* connection at the exact + point between the indexed-row count and the indexable-row count -- the + interleaving the daemon produces continuously. + + Mutation that fails this: remove the ``BEGIN``/``owns_snapshot`` arm. The + second count then reads a newer snapshot than the first, the totals + disagree by the racing row, and ``check_fts_readiness`` raises + "Search index is incomplete" against an archive that is in fact complete. + """ + with open_connection(searchable_db) as setup_conn: + setup_conn.execute("DELETE FROM fts_freshness_state WHERE surface = 'messages_fts'") + setup_conn.commit() + + real_status = fts_lifecycle.fts_index_status_async + counter = itertools.count() + raced = False + + async def racing_status(probe_conn: aiosqlite.Connection) -> Any: + nonlocal raced + status = await real_status(probe_conn) + with open_connection(searchable_db) as writer: + _seed_session(writer, f"conv-late-{next(counter)}", "late arriving block") + writer.commit() + raced = True + return status + + monkeypatch.setattr(fts_lifecycle, "fts_index_status_async", racing_status) + + async with aiosqlite.connect(searchable_db) as conn: + conn.row_factory = aiosqlite.Row + # No exception: the readiness probes and the MATCH share one snapshot. + await entry_point(conn, _TERM, limit=5) + + assert raced, "the count-only fallback never ran, so no snapshot straddling was exercised" From 642fab2bf547622ca5075a0abba7357d20c07fc1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Thu, 3 Sep 2026 21:26:30 +0200 Subject: [PATCH 27/47] fix(storage): converge the ops tier instead of refusing it ops.db is disposable and evolves by re-applying its idempotent additive DDL, so an absent or superseded identity stamp is a convergence input rather than a refusal. The derived-identity refusal now applies to the index tier only, which is rebuilt through the daemon route and must still refuse a foreign identity. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YGi8wLWGR2HYBh8p8fXFXz --- polylogue/storage/sqlite/archive_tiers/bootstrap.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index 4a76106550..944d7f14f7 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -548,8 +548,12 @@ def initialize_archive_database( try: current_version = int(conn.execute("PRAGMA user_version").fetchone()[0]) required_version = archive_tier_spec(tier).version if expected_version is None else expected_version + # ops.db is disposable and converges by re-applying its idempotent + # additive DDL, so an absent or superseded identity stamp is a + # convergence input rather than a refusal; index.db is rebuilt through + # the daemon route and must refuse a foreign identity here. derived_tier = None - if tier in (ArchiveTier.INDEX, ArchiveTier.OPS) and current_version != 0: + if tier is ArchiveTier.INDEX and current_version != 0: from polylogue.storage.sqlite.archive_tiers.schema_identity import DerivedTier derived_tier = DerivedTier(tier.value) From 0f79fe20a691dda88e12b9a3e5b60fbe410a121b Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:16:54 +0200 Subject: [PATCH 28/47] test(storage): cover stale ops identity convergence --- .../storage/test_derived_schema_identity.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/storage/test_derived_schema_identity.py b/tests/unit/storage/test_derived_schema_identity.py index a27a938900..c9c088d836 100644 --- a/tests/unit/storage/test_derived_schema_identity.py +++ b/tests/unit/storage/test_derived_schema_identity.py @@ -6,6 +6,7 @@ import aiosqlite import pytest +from polylogue.storage.sqlite.archive_tiers import ARCHIVE_VERSION_BY_TIER from polylogue.storage.sqlite.archive_tiers.archive import ArchiveStore from polylogue.storage.sqlite.archive_tiers.bootstrap import ( initialize_active_archive_root, @@ -104,6 +105,22 @@ def test_current_unstamped_ops_tier_is_adopted_before_identity_validation(tmp_pa assert read_schema_identity(conn, DerivedTier.OPS) == derived_schema_identity(DerivedTier.OPS) +def test_superseded_ops_identity_converges_to_the_current_schema(tmp_path: Path) -> None: + """Disposable ops state is rebuilt in place instead of blocking startup.""" + path = tmp_path / "ops.db" + with sqlite3.connect(path) as conn: + initialize_archive_tier(conn, ArchiveTier.OPS) + conn.execute("UPDATE schema_identity SET identity = 'from-another-runtime' WHERE tier = 'ops'") + conn.execute("PRAGMA user_version = 1") + conn.commit() + + initialize_archive_database(path, ArchiveTier.OPS) + + with sqlite3.connect(path) as conn: + assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == ARCHIVE_VERSION_BY_TIER[ArchiveTier.OPS] + assert read_schema_identity(conn, DerivedTier.OPS) == derived_schema_identity(DerivedTier.OPS) + + def test_canonical_sync_bootstrap_adopts_current_unstamped_index(tmp_path: Path) -> None: path = tmp_path / "index.db" with sqlite3.connect(path) as conn: From b2159cf77aee6e6c454de06ccb14574cf5eddf59 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 01:37:54 +0200 Subject: [PATCH 29/47] refactor: dissolve the insight readiness verdict into convergence debt Delete the eight-value insight readiness verdict enum, the last references to the retired insight_materialization marker table, and the duplicate async readiness builder whose only caller was a test. Insight readiness is now the ordinary convergence signal. Four status descriptors still gated on the dropped marker table and emitted count keys SessionInsightStatusSnapshot does not accept, so session_insight_status_sync raised KeyError on every call. Two import-time checks now require every emitted count to be a snapshot field, every table_key to be one the presence probe reports, and every referenced fallback key to be one some descriptor emits. InsightReadinessEntry carries structural facts (table_present, diverged, incomplete) that the export gate and capability mapping read directly. InsightReadinessReport reports converged plus debt_stages from the convergence_debt ledger; None means the ledger was unreadable, which is never success. The three CTE-derived surfaces the builder silently dropped are now reported, so a known insight name no longer yields an empty entry. Co-Authored-By: Claude Opus 5 --- .../no-insight-readiness-verdict.txt | 0 .../patterns/no-insight-readiness-verdict.yml | 10 + devtools/patterns/registry.yaml | 1 + polylogue/analysis/export_bundles.py | 46 +- polylogue/analysis/readiness.py | 495 ++---------------- polylogue/cli/commands/insights.py | 13 +- polylogue/readiness/capability.py | 25 +- polylogue/storage/derived/session/status.py | 38 ++ .../storage/sqlite/archive_tiers/archive.py | 131 ++--- .../storage/sqlite/archive_tiers/index.py | 1 - .../storage/sqlite/archive_tiers/write.py | 7 +- tests/unit/api/test_facade_contracts.py | 6 +- tests/unit/cli/test_insights.py | 7 +- .../unit/cli/test_insights_command_runtime.py | 6 +- .../unit/core/test_insight_export_bundles.py | 4 +- tests/unit/core/test_insight_readiness.py | 123 ++--- .../unit/core/test_insight_surface_parity.py | 177 +++++++ tests/unit/core/test_readiness_capability.py | 3 +- .../daemon/test_derived_stage_debt_surface.py | 127 +++++ tests/unit/insights/test_fallback_markers.py | 7 +- 20 files changed, 594 insertions(+), 633 deletions(-) create mode 100644 devtools/patterns/baselines/no-insight-readiness-verdict.txt create mode 100644 devtools/patterns/no-insight-readiness-verdict.yml create mode 100644 tests/unit/core/test_insight_surface_parity.py create mode 100644 tests/unit/daemon/test_derived_stage_debt_surface.py diff --git a/devtools/patterns/baselines/no-insight-readiness-verdict.txt b/devtools/patterns/baselines/no-insight-readiness-verdict.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/devtools/patterns/no-insight-readiness-verdict.yml b/devtools/patterns/no-insight-readiness-verdict.yml new file mode 100644 index 0000000000..2024fb1f37 --- /dev/null +++ b/devtools/patterns/no-insight-readiness-verdict.yml @@ -0,0 +1,10 @@ +id: no-insight-readiness-verdict +language: Python +severity: error +message: "Insight readiness is convergence debt, not a per-model verdict taxonomy" +rule: + any: + - pattern: "InsightReadinessVerdict = $$$" + - pattern: "verdict: InsightReadinessVerdict" + - pattern: "aggregate_verdict: $TYPE" + - pattern: "aggregate_verdict: $TYPE = $DEFAULT" diff --git a/devtools/patterns/registry.yaml b/devtools/patterns/registry.yaml index 8f524c3e0a..b8731d2656 100644 --- a/devtools/patterns/registry.yaml +++ b/devtools/patterns/registry.yaml @@ -7,3 +7,4 @@ rules: - {id: raw-subprocess-write, rule: raw-subprocess-write.yml, baseline: baselines/raw-subprocess-write.txt, owner: 0oyqi/ok65v/fhikb, status: pending} - {id: connection-lifecycle, rule: connection-lifecycle.yml, baseline: baselines/connection-lifecycle.txt, owner: a7xr.1/zqe52, status: pending} - {id: no-session-insight-ready-flag, rule: no-session-insight-ready-flag.yml, baseline: baselines/no-session-insight-ready-flag.txt, owner: polylogue-ajxy4, status: enforcing} + - {id: no-insight-readiness-verdict, rule: no-insight-readiness-verdict.yml, baseline: baselines/no-insight-readiness-verdict.txt, owner: polylogue-ajxy4, status: enforcing} diff --git a/polylogue/analysis/export_bundles.py b/polylogue/analysis/export_bundles.py index 6f9ea7fedb..3532afcc83 100644 --- a/polylogue/analysis/export_bundles.py +++ b/polylogue/analysis/export_bundles.py @@ -28,11 +28,6 @@ InsightExportFormat = Literal["jsonl"] INSIGHT_EXPORT_BUNDLE_VERSION = 1 -# Readiness verdicts that block an insight's rows from entering the bundle: the -# materialized read model is stale against its source, built by an incompatible -# materializer version, or its table is missing. ``degraded``/``partial`` rows are -# still exported (the rows themselves are valid, just heuristically weak/incomplete). -_NON_EXPORTABLE_VERDICTS: frozenset[str] = frozenset({"stale", "incompatible", "missing"}) DEFAULT_EXPORT_INSIGHTS: tuple[str, ...] = ( "session_profiles", "session_work_events", @@ -79,7 +74,7 @@ class InsightExportFileSummary(ArchiveInsightModel): file: str schema_file: str row_count: int = 0 - readiness_verdict: str | None = None + withheld_reason: str | None = None warnings: tuple[str, ...] = () errors: tuple[str, ...] = () @@ -199,7 +194,7 @@ def _write_readme(path: Path, manifest: InsightExportBundleManifest) -> None: ] for insight in manifest.insights: lines.append( - f"| `{insight.insight_name}` | {insight.row_count} | `{insight.readiness_verdict or '-'}` | `{insight.file}` |" + f"| `{insight.insight_name}` | {insight.row_count} | `{insight.withheld_reason or '-'}` | `{insight.file}` |" ) if manifest.warnings: lines.extend(["", "## Warnings", ""]) @@ -207,6 +202,26 @@ def _write_readme(path: Path, manifest: InsightExportBundleManifest) -> None: path.write_text("\n".join(lines) + "\n", encoding="utf-8") +def _withheld_reason(entry: object | None) -> str | None: + """Why an insight's rows must not enter a bundle, or ``None`` to export. + + Divergence is the only export blocker. Rows flagged as heuristically weak + (``degraded_count``) or merely incomplete are still exported: they are + valid rows, just not the whole picture. + """ + if entry is None: + return None + if not bool(getattr(entry, "table_present", True)): + return "insight table is absent" + if int(getattr(entry, "incompatible_count", 0) or 0): + return "insight rows fail their schema contract" + stale = int(getattr(entry, "stale_count", 0) or 0) + orphan = int(getattr(entry, "orphan_count", 0) or 0) + if stale or orphan: + return f"insight rows diverge from their sources (stale={stale} orphan={orphan})" + return None + + def _prepare_target(request: InsightExportBundleRequest) -> Path: target = request.output_path if target.exists() and not request.overwrite: @@ -256,14 +271,13 @@ async def export_insight_bundle( errors: list[str] = [] items: list[ArchiveInsightModel] = [] readiness_entry = readiness_by_name.get(insight_name) - verdict = readiness_entry.verdict if readiness_entry is not None else None - if verdict in _NON_EXPORTABLE_VERDICTS: - # The materialized read model does not reflect the current source - # (stale high-water mark), is built by an incompatible materializer - # version, or is absent. Exporting its rows would bundle untrustworthy - # data, so record the readiness failure and emit an empty file rather - # than silently shipping divergent insights. - errors.append(f"insight readiness verdict is '{verdict}'; rows withheld from export") + withheld_reason = _withheld_reason(readiness_entry) + if withheld_reason is not None: + # The rows do not reflect the sources they were built from: the + # table is absent, its rows outlive their sessions, or it fails + # its schema contract. Exporting them would bundle untrustworthy + # data, so emit an empty file and record why. + errors.append(f"{withheld_reason}; rows withheld from export") else: try: items = await fetch_insights_async(insight_type, operations, **kwargs) @@ -277,7 +291,7 @@ async def export_insight_bundle( file=insight_file, schema_file=schema_file, row_count=len(items), - readiness_verdict=verdict, + withheld_reason=withheld_reason, warnings=warnings, errors=tuple(errors), ) diff --git a/polylogue/analysis/readiness.py b/polylogue/analysis/readiness.py index ac77245f27..d48693573e 100644 --- a/polylogue/analysis/readiness.py +++ b/polylogue/analysis/readiness.py @@ -3,44 +3,14 @@ from __future__ import annotations from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Literal, cast -import aiosqlite from pydantic import Field from polylogue.analysis.archive_models import ARCHIVE_INSIGHT_CONTRACT_VERSION, ArchiveInsightModel -from polylogue.archive.query.spec import parse_query_date from polylogue.maintenance.targets import build_maintenance_target_catalog -from polylogue.storage.derived.session.runtime import SessionInsightStatusSnapshot -from polylogue.storage.introspection import table_exists_async as _table_exists _REPAIR_HINT = build_maintenance_target_catalog().repair_hint(("session_insights",), include_run_all=True) -InsightReadinessVerdict = Literal[ - "ready", "partial", "empty", "missing", "stale", "incompatible", "degraded", "unknown" -] - - -def _origin_value(origin: str | None) -> str | None: - from polylogue.storage.sqlite.archive_tiers.archive import _origin_value as _impl - - return _impl(origin) - - -def _readiness_query_ms(field: str, value: str | None) -> int | None: - parsed = parse_query_date(field, value) - if parsed is None: - return None - return int(parsed.timestamp() * 1000) - - -def _iso_from_ms(value: object) -> str | None: - if value is None: - return None - epoch_ms = int(cast("int | float | str", value)) - return datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc).isoformat() - class InsightReadinessQuery(ArchiveInsightModel): insights: tuple[str, ...] = () @@ -72,7 +42,7 @@ class InsightReadinessEntry(ArchiveInsightModel): insight_name: str display_name: str contract_version: int = ARCHIVE_INSIGHT_CONTRACT_VERSION - verdict: InsightReadinessVerdict = "unknown" + table_present: bool = True row_count: int = 0 expected_row_count: int | None = None missing_count: int = 0 @@ -90,139 +60,61 @@ class InsightReadinessEntry(ArchiveInsightModel): repair_command: str = _REPAIR_HINT evidence: tuple[str, ...] = () + @property + def diverged(self) -> bool: + """The stored rows do not reflect the sources they were built from. + + Divergence is the content comparison every derived object makes: the + table is absent, its rows outlive their session, or they were built + against a schema shape the reader cannot trust. Callers that must not + publish untrustworthy rows gate on this; callers reporting coverage + read the counts directly. + """ + return not self.table_present or bool(self.incompatible_count or self.stale_count or self.orphan_count) + + @property + def incomplete(self) -> bool: + """Sources exist that have no row yet -- ordinary convergence backlog.""" + return bool(self.missing_count) or ( + self.expected_row_count is not None and self.row_count < self.expected_row_count + ) + class InsightReadinessReport(ArchiveInsightModel): checked_at: str - aggregate_verdict: InsightReadinessVerdict total_sessions: int = 0 origin: str | None = None since: str | None = None until: str | None = None insights: tuple[InsightReadinessEntry, ...] = () + # Readiness is one signal: has convergence caught up. ``None`` means the + # debt ledger could not be read -- unknown is never success. ``debt_stages`` + # names the stages still holding retryable debt when it has not. + converged: bool | None = None + debt_stages: tuple[str, ...] = () @dataclass(frozen=True, slots=True) class InsightReadinessSpec: + """Public name and display label for one archive insight surface.""" + insight_name: str display_name: str - table_name: str | None - row_count_attr: str - expected_count_attr: str | None = None - missing_count_attr: str | None = None - missing_count_attrs: tuple[str, ...] = () - stale_count_attr: str | None = None - orphan_count_attr: str | None = None - artifacts: tuple[str, ...] = () - # insight tables are keyed by ``session_id`` and carry no - # ``source_name``/``source_updated_at``/``materializer_version`` columns of - # their own. Provider and time coverage derive from a join to ``sessions``; - # the join is enabled per spec because some specs (e.g. archive coverage - # over ``sessions`` itself) already expose those columns directly. - provider_via_session: bool = True - fallback_payload_columns: tuple[str, ...] = () - empty_is_ready: bool = False _SPECS: tuple[InsightReadinessSpec, ...] = ( - InsightReadinessSpec( - insight_name="session_profiles", - display_name="Session Profiles", - table_name="session_profiles", - row_count_attr="profile_row_count", - expected_count_attr="total_sessions", - missing_count_attr="missing_profile_row_count", - stale_count_attr="stale_profile_row_count", - orphan_count_attr="orphan_profile_row_count", - artifacts=("session_profiles",), - ), - InsightReadinessSpec( - insight_name="session_work_events", - display_name="Work Events", - table_name="session_work_events", - row_count_attr="work_event_inference_count", - expected_count_attr="expected_work_event_inference_count", - missing_count_attr=None, - stale_count_attr="stale_work_event_inference_count", - orphan_count_attr="orphan_work_event_inference_count", - artifacts=("session_work_events",), - fallback_payload_columns=("inference_json",), - ), - InsightReadinessSpec( - insight_name="session_phases", - display_name="Session Phases", - table_name="session_phases", - row_count_attr="phase_count", - expected_count_attr="expected_phase_count", - missing_count_attr=None, - stale_count_attr="stale_phase_count", - orphan_count_attr="orphan_phase_count", - artifacts=("session_phases",), - ), - # polylogue-dab/itvd: session_runs/session_observed_events/ - # session_context_snapshots are source-derived CTE relations - # (run_projection_relations.py), not tables, so they can never appear in - # sqlite_master. table_name points at the always-present `sessions` - # table purely so the presence gate (`table_present` in `_entry()`) - # reports true and the real row_count (already computed in `status` from - # the CTE) drives the verdict, instead of - # permanently reporting "missing". `artifacts` intentionally keeps the - # legacy table name -- it is genuinely, permanently absent, and - # InsightStorageArtifact.present reporting that honestly is useful - # diagnostic info distinguishing "no cache table" from "not ready". - InsightReadinessSpec( - insight_name="session_runs", - display_name="Session Runs", - table_name="sessions", - row_count_attr="run_count", - artifacts=("session_runs",), - empty_is_ready=True, - ), - InsightReadinessSpec( - insight_name="session_observed_events", - display_name="Observed Events", - table_name="sessions", - row_count_attr="observed_event_count", - artifacts=("session_observed_events",), - empty_is_ready=True, - ), - InsightReadinessSpec( - insight_name="session_context_snapshots", - display_name="Context Snapshots", - table_name="sessions", - row_count_attr="context_snapshot_count", - artifacts=("session_context_snapshots",), - empty_is_ready=True, - ), - InsightReadinessSpec( - insight_name="threads", - display_name="Work Threads", - table_name="threads", - row_count_attr="thread_count", - expected_count_attr="root_threads", - missing_count_attr=None, - stale_count_attr="stale_thread_count", - orphan_count_attr="orphan_thread_count", - artifacts=("threads",), - provider_via_session=False, - ), - InsightReadinessSpec( - insight_name="session_tag_rollups", - display_name="Session Tag Rollups", - table_name="session_tags", - row_count_attr="tag_rollup_count", - expected_count_attr="expected_tag_rollup_count", - stale_count_attr="stale_tag_rollup_count", - artifacts=("session_tags",), - ), - InsightReadinessSpec( - insight_name="archive_coverage", - display_name="Archive Coverage", - table_name="sessions", - row_count_attr="total_sessions", - artifacts=("sessions",), - ), + InsightReadinessSpec("session_profiles", "Session Profiles"), + InsightReadinessSpec("session_work_events", "Work Events"), + InsightReadinessSpec("session_phases", "Session Phases"), + InsightReadinessSpec("session_runs", "Session Runs"), + InsightReadinessSpec("session_observed_events", "Observed Events"), + InsightReadinessSpec("session_context_snapshots", "Context Snapshots"), + InsightReadinessSpec("threads", "Threads"), + InsightReadinessSpec("session_tag_rollups", "Session Tag Rollups"), + InsightReadinessSpec("archive_coverage", "Archive Coverage"), ) + _SPEC_BY_NAME = {spec.insight_name: spec for spec in _SPECS} _ALIASES = { **{spec.insight_name.replace("_", "-"): spec.insight_name for spec in _SPECS}, @@ -243,6 +135,11 @@ def known_insight_readiness_names() -> tuple[str, ...]: return tuple(spec.insight_name for spec in _SPECS) +def insight_display_name(name: str) -> str: + """Public display label for one insight readiness surface.""" + return _SPEC_BY_NAME[normalize_insight_readiness_name(name)].display_name + + def normalize_insight_readiness_name(value: str) -> str: normalized = value.strip().replace("-", "_") if normalized in _SPEC_BY_NAME: @@ -253,312 +150,6 @@ def normalize_insight_readiness_name(value: str) -> str: raise ValueError(f"Unknown insight readiness target: {value}") -def _count(status: SessionInsightStatusSnapshot, attr: str | None) -> int: - if attr is None: - return 0 - return int(getattr(status, attr)) - - -def _missing_count(status: SessionInsightStatusSnapshot, spec: InsightReadinessSpec) -> int: - return _count(status, spec.missing_count_attr) + sum(_count(status, attr) for attr in spec.missing_count_attrs) - - -def _entry_verdict( - *, - table_present: bool, - row_count: int, - expected_row_count: int | None, - missing_count: int, - stale_count: int, - orphan_count: int, - incompatible_count: int, - degraded_count: int, - empty_is_ready: bool = False, -) -> InsightReadinessVerdict: - if not table_present: - return "missing" - if incompatible_count: - return "incompatible" - if stale_count or orphan_count: - return "stale" - if missing_count or (expected_row_count is not None and row_count < expected_row_count): - return "partial" - if degraded_count: - return "degraded" - if row_count == 0: - return "ready" if empty_is_ready else "empty" - return "ready" - - -def _aggregate_verdict(entries: tuple[InsightReadinessEntry, ...]) -> InsightReadinessVerdict: - verdicts = {entry.verdict for entry in entries} - priority = ( - "incompatible", - "stale", - "partial", - "missing", - "degraded", - "unknown", - "empty", - ) - for verdict in priority: - if verdict in verdicts: - return verdict - return "ready" - - -async def _table_columns(conn: aiosqlite.Connection, table: str) -> set[str]: - rows = await (await conn.execute(f"PRAGMA table_info({table})")).fetchall() - return {str(row[1]) for row in rows} - - -def _normalize_origin_filter(origin: str | None) -> str | None: - if not origin: - return None - return _origin_value(origin) - - -def _where_clause( - spec: InsightReadinessSpec, - query: InsightReadinessQuery, -) -> tuple[str, list[object]]: - """Build a origin/time filter expressed against the joined ``sessions``. - - Archive insight tables key everything on ``session_id``; origin identity - lives on ``sessions.origin`` and recency on ``sessions.sort_key_ms``, so - every filter clause references the ``s.`` alias from the session join. - """ - clauses: list[str] = [] - params: list[object] = [] - origin = _normalize_origin_filter(query.origin) - if origin is not None: - clauses.append("s.origin = ?") - params.append(origin) - if query.since: - since_ms = _readiness_query_ms("since", query.since) - if since_ms is not None: - clauses.append("s.sort_key_ms >= ?") - params.append(since_ms) - if query.until: - until_ms = _readiness_query_ms("until", query.until) - if until_ms is not None: - clauses.append("s.sort_key_ms <= ?") - params.append(until_ms) - return (" WHERE " + " AND ".join(clauses), params) if clauses else ("", params) - - -async def _origin_coverage( - conn: aiosqlite.Connection, - spec: InsightReadinessSpec, - query: InsightReadinessQuery, - *, - table_present: bool, -) -> tuple[InsightOriginCoverage, ...]: - if not table_present or spec.table_name is None or not spec.provider_via_session: - return () - where, params = _where_clause(spec, query) - sql = ( - "SELECT s.origin AS origin, COUNT(*) AS row_count, " - "MIN(s.sort_key_ms) AS min_time_ms, MAX(s.sort_key_ms) AS max_time_ms " - f"FROM {spec.table_name} AS t " - "JOIN sessions AS s ON s.session_id = t.session_id" - f"{where} GROUP BY s.origin ORDER BY s.origin" - ) - rows = await (await conn.execute(sql, tuple(params))).fetchall() - return tuple( - InsightOriginCoverage( - origin=str(row["origin"]) if row["origin"] is not None else "unknown", - row_count=int(row["row_count"]), - min_time=_iso_from_ms(row["min_time_ms"]), - max_time=_iso_from_ms(row["max_time_ms"]), - ) - for row in rows - ) - - -async def _fallback_coverage( - conn: aiosqlite.Connection, - spec: InsightReadinessSpec, - *, - table_present: bool, - columns: set[str], -) -> tuple[int, dict[str, int]]: - """Count rows whose payload carries a non-empty ``fallback_reasons`` array. - - Returns ``(degraded_row_count, reason_totals)``. The row count is the - number of rows where at least one declared payload column reports any - fallback reason. ``reason_totals`` sums occurrences per reason across - every inspected payload column. The query uses ``json_extract`` and - ``json_each`` so each row contributes at most one count to - ``degraded_row_count`` regardless of how many payload columns flag it. - """ - - if not table_present or spec.table_name is None or not spec.fallback_payload_columns: - return (0, {}) - present_columns = tuple(column for column in spec.fallback_payload_columns if column in columns) - if not present_columns: - return (0, {}) - any_terms = " OR ".join( - f"json_array_length(COALESCE(json_extract({column}, '$.fallback_reasons'), '[]')) > 0" - for column in present_columns - ) - any_sql = f"SELECT COUNT(*) AS degraded FROM {spec.table_name} WHERE {any_terms}" - degraded_row = await (await conn.execute(any_sql)).fetchone() - degraded_row_count = int(degraded_row["degraded"]) if degraded_row is not None else 0 - - reason_totals: dict[str, int] = {} - for column in present_columns: - reason_sql = ( - f"SELECT value AS reason, COUNT(*) AS occurrences FROM {spec.table_name}, " - f"json_each(COALESCE(json_extract({column}, '$.fallback_reasons'), '[]')) GROUP BY value" - ) - rows = await (await conn.execute(reason_sql)).fetchall() - for row in rows: - reason = str(row["reason"]) - reason_totals[reason] = reason_totals.get(reason, 0) + int(row["occurrences"]) - return (degraded_row_count, dict(sorted(reason_totals.items()))) - - -def _schema_contract_issues(spec: InsightReadinessSpec, columns: set[str]) -> tuple[str, ...]: - """Report structural schema drift for an archive insight table. - - has no per-row version columns and derives origin/time - via the ``sessions`` join, so the only contract a present table can break - is its own primary ``session_id`` key (every archive insight table is keyed - on it). A missing ``session_id`` column means the table is not the expected - archive shape and its rows cannot be trusted. - """ - if spec.provider_via_session and "session_id" not in columns: - return (f"missing session_id column: {spec.table_name}",) - return () - - -def _evidence( - *, - row_count: int, - expected_row_count: int | None, - missing_count: int, - stale_count: int, - orphan_count: int, - incompatible_count: int, - degraded_count: int, - fallback_reason_counts: dict[str, int], - schema_contract_issues: tuple[str, ...], -) -> tuple[str, ...]: - values = [f"rows={row_count}"] - if expected_row_count is not None: - values.append(f"expected={expected_row_count}") - if missing_count: - values.append(f"missing={missing_count}") - if stale_count: - values.append(f"stale={stale_count}") - if orphan_count: - values.append(f"orphan={orphan_count}") - if incompatible_count: - values.append(f"incompatible={incompatible_count}") - if degraded_count: - values.append(f"degraded={degraded_count}") - values.extend(f"fallback_reason={reason}={count}" for reason, count in fallback_reason_counts.items()) - values.extend(f"schema_issue={issue}" for issue in schema_contract_issues) - return tuple(values) - - -async def _entry( - conn: aiosqlite.Connection, - status: SessionInsightStatusSnapshot, - spec: InsightReadinessSpec, - query: InsightReadinessQuery, -) -> InsightReadinessEntry: - table_present = bool(spec.table_name and await _table_exists(conn, spec.table_name)) - row_count = _count(status, spec.row_count_attr) - expected_row_count = _count(status, spec.expected_count_attr) if spec.expected_count_attr is not None else None - missing_count = _missing_count(status, spec) - stale_count = _count(status, spec.stale_count_attr) - orphan_count = _count(status, spec.orphan_count_attr) - columns = await _table_columns(conn, spec.table_name) if table_present and spec.table_name is not None else set() - schema_contract_issues = _schema_contract_issues(spec, columns) if table_present else () - version_coverage: tuple[InsightVersionCoverage, ...] = () - incompatible_count = row_count if schema_contract_issues else 0 - origin_coverage = await _origin_coverage(conn, spec, query, table_present=table_present) - degraded_count, fallback_reason_counts = await _fallback_coverage( - conn, spec, table_present=table_present, columns=columns - ) - artifacts: list[InsightStorageArtifact] = [] - for artifact in spec.artifacts: - artifacts.append( - InsightStorageArtifact( - name=artifact, - present=await _table_exists(conn, artifact), - ) - ) - min_time = min((item.min_time for item in origin_coverage if item.min_time), default=None) - max_time = max((item.max_time for item in origin_coverage if item.max_time), default=None) - verdict = _entry_verdict( - table_present=table_present, - row_count=row_count, - expected_row_count=expected_row_count, - missing_count=missing_count, - stale_count=stale_count, - orphan_count=orphan_count, - incompatible_count=incompatible_count, - degraded_count=degraded_count, - empty_is_ready=spec.empty_is_ready, - ) - return InsightReadinessEntry( - insight_name=spec.insight_name, - display_name=spec.display_name, - verdict=verdict, - row_count=row_count, - expected_row_count=expected_row_count, - missing_count=missing_count, - stale_count=stale_count, - orphan_count=orphan_count, - incompatible_count=incompatible_count, - degraded_count=degraded_count, - fallback_reason_counts=fallback_reason_counts, - storage_artifacts=tuple(artifacts), - origin_coverage=origin_coverage, - version_coverage=version_coverage, - schema_contract_issues=schema_contract_issues, - min_time=min_time, - max_time=max_time, - evidence=_evidence( - row_count=row_count, - expected_row_count=expected_row_count, - missing_count=missing_count, - stale_count=stale_count, - orphan_count=orphan_count, - incompatible_count=incompatible_count, - degraded_count=degraded_count, - fallback_reason_counts=fallback_reason_counts, - schema_contract_issues=schema_contract_issues, - ), - ) - - -async def build_insight_readiness_report( - conn: aiosqlite.Connection, - status: SessionInsightStatusSnapshot, - query: InsightReadinessQuery | None = None, -) -> InsightReadinessReport: - request = query or InsightReadinessQuery() - selected = tuple(normalize_insight_readiness_name(insight) for insight in request.insights) - specs = tuple(_SPEC_BY_NAME[name] for name in selected) if selected else _SPECS - entries: list[InsightReadinessEntry] = [] - for spec in specs: - entries.append(await _entry(conn, status, spec, request)) - insights = tuple(entries) - return InsightReadinessReport( - checked_at=datetime.now(timezone.utc).isoformat(), - aggregate_verdict=_aggregate_verdict(insights), - total_sessions=status.total_sessions, - origin=request.origin, - since=request.since, - until=request.until, - insights=insights, - ) - - __all__ = [ "InsightOriginCoverage", "InsightReadinessEntry", @@ -566,7 +157,7 @@ async def build_insight_readiness_report( "InsightReadinessReport", "InsightStorageArtifact", "InsightVersionCoverage", - "build_insight_readiness_report", + "insight_display_name", "known_insight_readiness_names", "normalize_insight_readiness_name", ] diff --git a/polylogue/cli/commands/insights.py b/polylogue/cli/commands/insights.py index 1e585097db..d90803fee1 100644 --- a/polylogue/cli/commands/insights.py +++ b/polylogue/cli/commands/insights.py @@ -206,7 +206,12 @@ def _render_status_plain(report: InsightReadinessReport) -> None: def origin_label(value: str | None) -> str: return value or "-" - click.echo(f"Insight Readiness: {report.aggregate_verdict}") + if report.converged is None: + click.echo("Convergence: unknown (debt ledger unreadable)") + elif report.converged: + click.echo("Convergence: caught up") + else: + click.echo(f"Convergence: debt in {', '.join(report.debt_stages)}") click.echo(f"Total sessions: {report.total_sessions}") if report.origin or report.since or report.until: click.echo( @@ -215,7 +220,8 @@ def origin_label(value: str | None) -> str: click.echo("") for insight in report.insights: expected = f" expected={insight.expected_row_count}" if insight.expected_row_count is not None else "" - click.echo(f"{insight.insight_name}: {insight.verdict} rows={insight.row_count}{expected}") + presence = "" if insight.table_present else " (table absent)" + click.echo(f"{insight.insight_name}: rows={insight.row_count}{expected}{presence}") if insight.missing_count or insight.stale_count or insight.orphan_count or insight.incompatible_count: click.echo( " " @@ -240,7 +246,8 @@ def _render_export_plain(result: InsightExportBundleResult) -> None: click.echo(f"Coverage: {result.coverage_path}") click.echo("") for insight in result.manifest.insights: - click.echo(f"{insight.insight_name}: rows={insight.row_count} readiness={insight.readiness_verdict or '-'}") + withheld = f" withheld={insight.withheld_reason}" if insight.withheld_reason else "" + click.echo(f"{insight.insight_name}: rows={insight.row_count}{withheld}") for warning in insight.warnings: click.echo(f" warning: {warning}") for error in insight.errors: diff --git a/polylogue/readiness/capability.py b/polylogue/readiness/capability.py index ef4709d9bb..883a7f1182 100644 --- a/polylogue/readiness/capability.py +++ b/polylogue/readiness/capability.py @@ -808,17 +808,20 @@ def component_from_archive_surface( def component_from_insight_entry(entry: Any, *, scope: str = "insights") -> ComponentReadiness: - verdict = str(getattr(entry, "verdict", "unknown")) - state = { - "ready": CapabilityReadinessState.READY, - "partial": CapabilityReadinessState.DEGRADED, - "empty": CapabilityReadinessState.MISSING, - "missing": CapabilityReadinessState.MISSING, - "stale": CapabilityReadinessState.STALE, - "incompatible": CapabilityReadinessState.POISONED, - "degraded": CapabilityReadinessState.DEGRADED, - "unknown": CapabilityReadinessState.UNKNOWN, - }.get(verdict, CapabilityReadinessState.UNKNOWN) + # Derived from the entry's own counts. There is no insight-private verdict + # taxonomy: rows that outlive their source or fail their schema contract are + # untrustworthy, rows a source is still waiting on are backlog, everything + # else is ready. + if not bool(getattr(entry, "table_present", True)): + state = CapabilityReadinessState.MISSING + elif int(getattr(entry, "incompatible_count", 0) or 0): + state = CapabilityReadinessState.POISONED + elif int(getattr(entry, "stale_count", 0) or 0) or int(getattr(entry, "orphan_count", 0) or 0): + state = CapabilityReadinessState.STALE + elif bool(getattr(entry, "incomplete", False)) or int(getattr(entry, "degraded_count", 0) or 0): + state = CapabilityReadinessState.DEGRADED + else: + state = CapabilityReadinessState.READY return ComponentReadiness( component=str(getattr(entry, "insight_name", "unknown")), scope=scope, diff --git a/polylogue/storage/derived/session/status.py b/polylogue/storage/derived/session/status.py index d2727733b5..8fdfe10f0b 100644 --- a/polylogue/storage/derived/session/status.py +++ b/polylogue/storage/derived/session/status.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses import sqlite3 from dataclasses import dataclass from typing import TypeAlias @@ -524,6 +525,43 @@ async def _stale_session_profile_count_sql_async(conn: aiosqlite.Connection) -> ), ) +# The status counts are splatted into ``SessionInsightStatusSnapshot``, and +# descriptors index ``tables``/``counts`` by name. Every emitted key must +# therefore be a snapshot field, every ``table_key`` a table the presence probe +# reports, and every referenced fallback/source key one some descriptor emits. +# A key that satisfies none of these raises only when a status call reaches it, +# so all three are checked at import. +_SNAPSHOT_COUNT_FIELDS = frozenset(field.name for field in dataclasses.fields(SessionInsightStatusSnapshot)) +_TABLE_PRESENCE_KEYS = frozenset(descriptor.key for descriptor in _TABLE_DESCRIPTORS) +_EMITTED_COUNT_KEYS = ( + {"total_sessions", "root_threads"} + | {descriptor.count_key for descriptor in _TABLE_DESCRIPTORS if descriptor.count_key is not None} + | {descriptor.count_key for descriptor in _FTS_DESCRIPTORS} + | {descriptor.duplicate_count_key for descriptor in _FTS_DESCRIPTORS} + | {descriptor.count_key for descriptor in _COUNT_DESCRIPTORS} +) + +if _unknown_counts := sorted(_EMITTED_COUNT_KEYS - _SNAPSHOT_COUNT_FIELDS): + raise RuntimeError(f"status descriptors emit counts absent from SessionInsightStatusSnapshot: {_unknown_counts}") + +if _unknown_tables := sorted( + ( + {descriptor.table_key for descriptor in _COUNT_DESCRIPTORS if descriptor.table_key is not None} + | {descriptor.table_key for descriptor in _FTS_DESCRIPTORS} + ) + - _TABLE_PRESENCE_KEYS +): + raise RuntimeError(f"status descriptors gate on tables the presence probe does not report: {_unknown_tables}") + +if _unknown_references := sorted( + ( + {descriptor.fallback_count_key for descriptor in _COUNT_DESCRIPTORS if descriptor.fallback_count_key} + | {descriptor.source_count_key for descriptor in _FTS_DESCRIPTORS} + ) + - _EMITTED_COUNT_KEYS +): + raise RuntimeError(f"status descriptors reference counts nothing emits: {_unknown_references}") + def _to_int(row: tuple[object, ...] | sqlite3.Row | None) -> int: if not row: diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 4d4fc2944d..9e0274ea1f 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -77,9 +77,9 @@ InsightReadinessEntry, InsightReadinessQuery, InsightReadinessReport, - InsightReadinessVerdict, InsightStorageArtifact, InsightVersionCoverage, + insight_display_name, known_insight_readiness_names, normalize_insight_readiness_name, ) @@ -5288,9 +5288,11 @@ def insight_readiness_report(self, query: InsightReadinessQuery | None = None) - ) is not None ) + converged, debt_stages = self._derived_convergence_signal() return InsightReadinessReport( checked_at=datetime.now(UTC).isoformat(), - aggregate_verdict=_insight_readiness_aggregate_verdict(entries), + converged=converged, + debt_stages=debt_stages, total_sessions=total_sessions, origin=request.origin, since=request.since, @@ -5419,6 +5421,33 @@ def _archive_fallback_coverage( reason_totals[reason] = reason_totals.get(reason, 0) + int(row["occurrences"]) return (degraded_count, dict(sorted(reason_totals.items()))) + def _derived_convergence_signal(self) -> tuple[bool | None, tuple[str, ...]]: + """Report whether convergence has caught up, and which stages have not. + + Derived rows have no lifecycle of their own: their readiness is exactly + the ordinary convergence signal. ops.db is disposable, so a ledger that + cannot be read reports ``None`` -- unknown, never converged. + """ + if not self.ops_db_path.exists(): + return (None, ()) + try: + conn = sqlite3.connect(f"file:{self.ops_db_path}?mode=ro", uri=True) + try: + present = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'convergence_debt'" + ).fetchone() + if present is None: + return (None, ()) + rows = conn.execute( + "SELECT DISTINCT stage FROM convergence_debt WHERE status IN ('failed', 'deferred') ORDER BY stage" + ).fetchall() + finally: + conn.close() + except sqlite3.Error: + return (None, ()) + stages = tuple(str(row[0]) for row in rows) + return (not stages, stages) + def _insight_readiness_entry( self, name: str, @@ -5432,7 +5461,6 @@ def _insight_readiness_entry( ) -> InsightReadinessEntry | None: specs = { "session_profiles": ( - "Session Profiles", "session_profiles", status.profile_row_count, total_sessions, @@ -5442,7 +5470,6 @@ def _insight_readiness_entry( ("session_profiles",), ), "session_work_events": ( - "Work Events", "session_work_events", status.work_event_inference_count, status.expected_work_event_inference_count, @@ -5452,7 +5479,6 @@ def _insight_readiness_entry( ("session_work_events",), ), "session_phases": ( - "Session Phases", "session_phases", status.phase_count, status.expected_phase_count, @@ -5461,8 +5487,40 @@ def _insight_readiness_entry( status.orphan_phase_count, ("session_phases",), ), + # polylogue-dab/itvd: runs, observed events and context snapshots are + # source-derived CTE relations, never tables, so they can never appear + # in sqlite_master. The presence probe reads `sessions` (always + # present) and the real count comes from the status snapshot; + # `artifacts` keeps the legacy table name because reporting it + # permanently absent distinguishes "no cache table" from "no rows". + "session_runs": ( + "sessions", + status.run_count, + None, + 0, + 0, + 0, + ("session_runs",), + ), + "session_observed_events": ( + "sessions", + status.observed_event_count, + None, + 0, + 0, + 0, + ("session_observed_events",), + ), + "session_context_snapshots": ( + "sessions", + status.context_snapshot_count, + None, + 0, + 0, + 0, + ("session_context_snapshots",), + ), "threads": ( - "Threads", "threads", status.thread_count, status.root_threads, @@ -5472,7 +5530,6 @@ def _insight_readiness_entry( ("threads", "thread_sessions"), ), "session_tag_rollups": ( - "Session Tag Rollups", "session_tags", status.tag_rollup_count, status.expected_tag_rollup_count, @@ -5482,7 +5539,6 @@ def _insight_readiness_entry( ("session_tags",), ), "archive_coverage": ( - "Archive Coverage", "sessions", total_sessions, total_sessions, @@ -5496,7 +5552,6 @@ def _insight_readiness_entry( if spec is None: return None ( - display_name, table_name, row_count, expected_row_count, @@ -5527,21 +5582,10 @@ def _insight_readiness_entry( since_ms=since_ms, until_ms=until_ms, ) - verdict = _archive_insight_readiness_verdict( - table_present=table_present, - row_count=row_count, - expected_row_count=expected_row_count, - missing_count=missing_count, - stale_count=stale_count, - orphan_count=orphan_count, - incompatible_count=incompatible_count, - degraded_count=degraded_count, - total_sessions=total_sessions, - ) return InsightReadinessEntry( insight_name=name, - display_name=display_name, - verdict=verdict, + display_name=insight_display_name(name), + table_present=table_present, row_count=row_count, expected_row_count=expected_row_count, missing_count=missing_count, @@ -7882,49 +7926,6 @@ def _epoch_ms_from_iso(value: object) -> int | None: } -def _archive_insight_readiness_verdict( - *, - table_present: bool, - row_count: int, - expected_row_count: int | None, - missing_count: int, - stale_count: int, - orphan_count: int, - incompatible_count: int, - degraded_count: int, - total_sessions: int, -) -> InsightReadinessVerdict: - if not table_present: - return "missing" - if incompatible_count: - return "incompatible" - if stale_count or orphan_count: - return "stale" - if missing_count or (expected_row_count is not None and row_count < expected_row_count): - return "partial" - if row_count == 0: - # An empty archive (no sessions at all) reports every surface as empty. - # In a populated archive a surface with 0 expected rows is vacuously - # ready (e.g. no tags to roll up); a surface that should hold rows was - # already caught by the partial branch above. - if total_sessions > 0 and expected_row_count == 0: - return "ready" - return "empty" - if degraded_count: - return "degraded" - return "ready" - - -def _insight_readiness_aggregate_verdict( - entries: tuple[InsightReadinessEntry, ...], -) -> InsightReadinessVerdict: - verdicts = {entry.verdict for entry in entries} - for verdict in ("incompatible", "stale", "partial", "missing", "degraded", "unknown", "empty"): - if verdict in verdicts: - return verdict - return "ready" - - def _archive_insight_readiness_evidence( *, row_count: int, diff --git a/polylogue/storage/sqlite/archive_tiers/index.py b/polylogue/storage/sqlite/archive_tiers/index.py index ccb138a4ba..bc46f0d5a8 100644 --- a/polylogue/storage/sqlite/archive_tiers/index.py +++ b/polylogue/storage/sqlite/archive_tiers/index.py @@ -467,7 +467,6 @@ ) STRICT; """ -# ddl-lifecycle-waiver: derived CREATE TABLE insight_materialization retirement removes the obsolete marker; ordinary convergence regenerates surviving rows. INDEX_DDL = f""" {DERIVED_SCHEMA_META_DDL} diff --git a/polylogue/storage/sqlite/archive_tiers/write.py b/polylogue/storage/sqlite/archive_tiers/write.py index 80f8c5a6d2..aafa988752 100644 --- a/polylogue/storage/sqlite/archive_tiers/write.py +++ b/polylogue/storage/sqlite/archive_tiers/write.py @@ -6706,9 +6706,10 @@ def _set_edge( t0 = time.perf_counter() _refresh_session_counts(conn, child_session_id) # Late-parent resolution mutates an already-materialized child outside its - # own write path. Rebuild every projection whose input rows just changed; - # otherwise incremental convergence can retain the pre-extraction prefix - # in usage, delegation, and insight products indefinitely. + # own write path, so usage must be rebuilt here: it is aggregated at write + # time and nothing else revisits it. Derived session rows converge on their + # own -- the refreshed counts move the child's high-water mark, which is + # what the staleness comparison reads. conn.execute("DELETE FROM session_model_usage WHERE session_id = ?", (child_session_id,)) _aggregate_message_tokens_into_model_usage(conn, child_session_id) _aggregate_provider_usage_into_model_usage(conn, child_session_id) diff --git a/tests/unit/api/test_facade_contracts.py b/tests/unit/api/test_facade_contracts.py index abc04c0ea1..120ef43de5 100644 --- a/tests/unit/api/test_facade_contracts.py +++ b/tests/unit/api/test_facade_contracts.py @@ -5249,9 +5249,9 @@ async def test_archive_tiers_api_threads_read_index_tier(tmp_path: Path) -> None assert candidates[0].file_overlap == ("/realm/project/polylogue/polylogue/api/archive.py",) readiness_by_name = {entry.insight_name: entry for entry in readiness.insights} assert readiness.total_sessions == 2 - assert readiness_by_name["session_profiles"].verdict == "stale" + assert readiness_by_name["session_profiles"].diverged assert readiness_by_name["session_profiles"].missing_count == 1 - assert readiness_by_name["threads"].verdict == "stale" + assert readiness_by_name["threads"].diverged assert readiness_by_name["threads"].row_count == 1 assert readiness_by_name["threads"].expected_row_count == 1 assert readiness_by_name["threads"].stale_count == 1 @@ -5275,7 +5275,7 @@ async def test_archive_tiers_api_threads_read_index_tier(tmp_path: Path) -> None assert coverage["total_sessions"] == 2 assert exported_threads == [] threads_summary = next(entry for entry in manifest["insights"] if entry["insight_name"] == "threads") - assert threads_summary["readiness_verdict"] == "stale" + assert "diverge" in threads_summary["withheld_reason"] assert threads_summary["row_count"] == 0 assert any("withheld" in error for error in threads_summary["errors"]) assert (export_target / "schemas" / "threads.schema.json").exists() diff --git a/tests/unit/cli/test_insights.py b/tests/unit/cli/test_insights.py index 49a8bf816b..6bc4f67dde 100644 --- a/tests/unit/cli/test_insights.py +++ b/tests/unit/cli/test_insights.py @@ -533,7 +533,10 @@ def test_insights_status_json(cli_workspace: CliWorkspace) -> None: assert result.exit_code == 0 payload = extract_json_result(result.output) - assert payload["aggregate_verdict"] == "degraded" + # The seeded workspace has an ops tier with an empty debt ledger: readiness + # is exactly "convergence has caught up". + assert payload["converged"] is True + assert payload["debt_stages"] == [] insights = {item["insight_name"]: item for item in json_object_list(payload["insights"])} assert set(insights) >= { "session_profiles", @@ -543,7 +546,7 @@ def test_insights_status_json(cli_workspace: CliWorkspace) -> None: "session_tag_rollups", "archive_coverage", } - assert insights["session_profiles"]["verdict"] == "degraded" + assert insights["session_profiles"]["table_present"] is True assert json_int(insights["session_profiles"]["degraded_count"]) == 2 assert json_int(insights["session_work_events"]["row_count"]) >= 1 diff --git a/tests/unit/cli/test_insights_command_runtime.py b/tests/unit/cli/test_insights_command_runtime.py index c3e3aad10c..1ab5c327a4 100644 --- a/tests/unit/cli/test_insights_command_runtime.py +++ b/tests/unit/cli/test_insights_command_runtime.py @@ -85,7 +85,8 @@ def _command_callback(command: click.Command) -> Callable[..., object]: def _status_report() -> InsightReadinessReport: return InsightReadinessReport( checked_at="2026-04-23T00:00:00+00:00", - aggregate_verdict="partial", + converged=False, + debt_stages=("derived",), total_sessions=10, origin="codex-session", since="2026-04-01", @@ -94,7 +95,6 @@ def _status_report() -> InsightReadinessReport: InsightReadinessEntry( insight_name="session_profiles", display_name="Session Profiles", - verdict="partial", row_count=7, expected_row_count=10, missing_count=1, @@ -128,7 +128,7 @@ def _export_result(tmp_path: Path) -> InsightExportBundleResult: file="insights/session_profiles.jsonl", schema_file="schemas/session_profiles.schema.json", row_count=7, - readiness_verdict="partial", + withheld_reason=None, warnings=("stale rows",), errors=("schema drift",), ), diff --git a/tests/unit/core/test_insight_export_bundles.py b/tests/unit/core/test_insight_export_bundles.py index f79844ec8b..61159d7517 100644 --- a/tests/unit/core/test_insight_export_bundles.py +++ b/tests/unit/core/test_insight_export_bundles.py @@ -160,7 +160,9 @@ async def test_insight_export_bundle_records_stale_readiness(cli_workspace: dict coverage = _json_file(target / "coverage.json") coverage_products = coverage["insights"] assert isinstance(coverage_products, list) - assert coverage_products[0]["verdict"] == "stale" + # Serialized coverage carries the divergence facts, not a verdict label. + assert coverage_products[0]["table_present"] is True + assert coverage_products[0]["stale_count"] >= 1 manifest = _json_file(target / "manifest.json") manifest_products = manifest["insights"] assert isinstance(manifest_products, list) diff --git a/tests/unit/core/test_insight_readiness.py b/tests/unit/core/test_insight_readiness.py index 90aaf223df..bc340a0ee7 100644 --- a/tests/unit/core/test_insight_readiness.py +++ b/tests/unit/core/test_insight_readiness.py @@ -2,20 +2,16 @@ from __future__ import annotations -import sqlite3 from pathlib import Path -import aiosqlite import pytest from polylogue.analysis.readiness import ( InsightReadinessEntry, InsightReadinessQuery, InsightReadinessReport, - build_insight_readiness_report, ) from polylogue.api import Polylogue -from polylogue.storage.derived.session.status import session_insight_status_sync from polylogue.storage.runtime.store_constants import SESSION_INSIGHT_MATERIALIZER_VERSION from tests.infra.storage_records import SessionBuilder @@ -69,9 +65,8 @@ async def test_insight_readiness_report_marks_rebuilt_insights_ready(cli_workspa report = await archive.insight_readiness_report() # The sparse seed deliberately lacks the evidence needed for a fully - # grounded profile. Rebuild is complete, but readiness must surface its - # fallback rather than falsely claiming a ready insight. - assert report.aggregate_verdict == "degraded" + # grounded profile. Rebuild is complete, so the rows do not diverge from + # their sources, but coverage must still surface the fallback. assert {insight.insight_name for insight in report.insights} >= { "session_profiles", "session_work_events", @@ -81,7 +76,7 @@ async def test_insight_readiness_report_marks_rebuilt_insights_ready(cli_workspa "archive_coverage", } profile = _entry_by_name(report, "session_profiles") - assert profile.verdict == "degraded" + assert not profile.diverged assert profile.degraded_count == 1 assert profile.fallback_reason_counts assert profile.row_count == 1 @@ -95,8 +90,9 @@ async def test_insight_readiness_report_marks_empty_insights(cli_workspace: dict report = await archive.insight_readiness_report(InsightReadinessQuery(insights=("session_profiles",))) profile = _entry_by_name(report, "session_profiles") - assert report.aggregate_verdict == "empty" - assert profile.verdict == "empty" + assert profile.table_present + assert not profile.diverged + assert not profile.incomplete assert profile.row_count == 0 assert profile.expected_row_count == 0 @@ -128,7 +124,9 @@ async def test_insight_readiness_report_marks_partial_and_incompatible_insights( archive = Polylogue(archive_root=cli_workspace["archive_root"], db_path=db_path) partial = await archive.insight_readiness_report(InsightReadinessQuery(insights=("session_profiles",))) - assert _entry_by_name(partial, "session_profiles").verdict == "partial" + incomplete = _entry_by_name(partial, "session_profiles") + assert incomplete.incomplete + assert not incomplete.diverged await _rebuild(db_path) with sqlite3.connect(db_path) as conn: @@ -140,7 +138,7 @@ async def test_insight_readiness_report_marks_partial_and_incompatible_insights( stale = await archive.insight_readiness_report(InsightReadinessQuery(insights=("session_profiles",))) profile = _entry_by_name(stale, "session_profiles") - assert profile.verdict == "stale" + assert profile.diverged assert profile.stale_count == 2 @@ -165,64 +163,53 @@ async def test_insight_readiness_report_marks_stale_insights(cli_workspace: dict report = await archive.insight_readiness_report(InsightReadinessQuery(insights=("session_profiles",))) profile = _entry_by_name(report, "session_profiles") - assert report.aggregate_verdict == "stale" - assert profile.verdict == "stale" + assert profile.diverged assert profile.stale_count == 1 +def test_absent_table_is_divergence_not_an_honest_zero() -> None: + """``table_present`` decides divergence; an empty present table does not. + + This is the derivation that replaced the ``missing``/``empty`` verdicts. + Goes red if ``diverged`` stops reading ``table_present`` -- an absent table + would then be indistinguishable from a table that legitimately holds no rows. + """ + absent = InsightReadinessEntry( + insight_name="session_profiles", + display_name="Session Profiles", + table_present=False, + ) + empty = InsightReadinessEntry( + insight_name="session_profiles", + display_name="Session Profiles", + table_present=True, + expected_row_count=0, + ) + + assert absent.diverged + assert not empty.diverged + assert not empty.incomplete + assert absent.row_count == empty.row_count == 0 + + @pytest.mark.asyncio -async def test_insight_readiness_report_marks_missing_insight_tables(tmp_path: Path) -> None: - db_path = tmp_path / "missing.db" - with sqlite3.connect(db_path) as conn: - conn.row_factory = sqlite3.Row - conn.executescript( - """ - CREATE TABLE sessions ( - session_id TEXT PRIMARY KEY, - parent_session_id TEXT, - source_name TEXT, - origin TEXT, - branch_type TEXT, - title TEXT, - git_branch TEXT, - native_id TEXT, - message_count INTEGER, - tool_use_count INTEGER, - created_at_ms INTEGER, - updated_at_ms INTEGER, - sort_key REAL, - updated_at TEXT - ); - CREATE TABLE blocks ( - block_id TEXT PRIMARY KEY, - session_id TEXT, - block_type TEXT, - message_id TEXT, - position INTEGER, - semantic_type TEXT, - tool_command TEXT, - tool_id TEXT, - tool_name TEXT, - tool_result_exit_code INTEGER, - tool_result_is_error INTEGER, - tool_outcome TEXT, - search_text TEXT - ); - INSERT INTO sessions (session_id, parent_session_id, source_name, sort_key, updated_at) - VALUES ('missing-root', NULL, 'codex', 1.0, '2026-04-01T00:00:00Z'); - """ - ) - status = session_insight_status_sync(conn) - - async with aiosqlite.connect(db_path) as conn: - conn.row_factory = aiosqlite.Row - report = await build_insight_readiness_report( - conn, - status, - InsightReadinessQuery(insights=("session_profiles",)), - ) +async def test_permanently_absent_storage_artifact_reports_absent(cli_workspace: dict[str, Path]) -> None: + """The presence probe reads sqlite_master rather than assuming presence. + + ``session_runs`` is a source-derived CTE relation whose legacy table never + exists, so its storage artifact must report ``present=False`` while the + surface itself still counts rows. Goes red if artifact presence is + hardcoded true, which would hide a genuinely missing table. + """ + db_path = cli_workspace["db_path"] + _seed_readiness_sessions(db_path) + await _rebuild(db_path) - profile = _entry_by_name(report, "session_profiles") - assert profile.verdict == "missing" - assert profile.row_count == 0 - assert profile.expected_row_count == 1 + archive = Polylogue(archive_root=cli_workspace["archive_root"], db_path=db_path) + report = await archive.insight_readiness_report(InsightReadinessQuery(insights=("session_runs",))) + + runs = _entry_by_name(report, "session_runs") + assert [artifact.name for artifact in runs.storage_artifacts] == ["session_runs"] + assert runs.storage_artifacts[0].present is False + # The surface is still reported; only its legacy cache table is absent. + assert runs.table_present diff --git a/tests/unit/core/test_insight_surface_parity.py b/tests/unit/core/test_insight_surface_parity.py new file mode 100644 index 0000000000..299d540342 --- /dev/null +++ b/tests/unit/core/test_insight_surface_parity.py @@ -0,0 +1,177 @@ +"""CLI, MCP and API project one insight list payload. + +Every insight list surface must reach its rows through the same product route +and wrap them in the same envelope. This harness reads all three for each +registered insight type against one seeded archive and requires the serialized +payloads to be identical. + +Anti-vacuity: goes red if any surface grows its own projection -- a CLI +renderer that reshapes rows, an MCP path that bypasses ``fetch_insights_async`` +and hand-builds an envelope, or a default (limit, offset) that drifts between +the Click parameter and the MCP tool signature. It also fails if the seeded +archive yields no rows for every insight type, which would make the comparison +vacuous. + +Only per-call provenance stamps are normalized away; every other field, +including row order and the ``{: [...], "total": N}`` envelope, is +compared exactly. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from polylogue.analysis.registry import INSIGHT_REGISTRY, InsightType, fetch_insights_async, insight_items_payload +from polylogue.api import Polylogue +from polylogue.cli.click_app import cli +from polylogue.mcp.insight_tool_contracts import InsightListToolSpec +from tests.infra.json_contracts import extract_json_result +from tests.infra.storage_records import SessionBuilder + + +def _parity_insight_types() -> list[InsightType]: + """Insight types every one of the three surfaces exposes.""" + return sorted( + ( + insight_type + for insight_type in INSIGHT_REGISTRY.values() + if insight_type.cli_command_name and insight_type.operations_method_name and insight_type.query_model + ), + key=lambda insight_type: insight_type.name, + ) + + +def _seed(db_path: Path) -> None: + ( + SessionBuilder(db_path, "parity-root") + .provider("claude-code") + .title("Parity Root") + .created_at("2026-03-01T10:00:00+00:00") + .updated_at("2026-03-01T10:10:00+00:00") + .add_message( + "u1", + role="user", + text="Inspect the parity harness and edit it.", + timestamp="2026-03-01T10:00:00+00:00", + ) + .add_message( + "a1", + role="assistant", + text="Reading and editing the harness.", + timestamp="2026-03-01T10:05:00+00:00", + blocks=[ + { + "type": "tool_use", + "tool_name": "Read", + "semantic_type": "file_read", + "input": {"path": "/workspace/polylogue/README.md"}, + }, + { + "type": "tool_use", + "tool_name": "Edit", + "semantic_type": "file_edit", + "input": {"path": "/workspace/polylogue/README.md"}, + }, + ], + ) + .save() + ) + ( + SessionBuilder(db_path, "parity-child") + .provider("codex") + .title("Parity Child") + .created_at("2026-03-01T11:00:00+00:00") + .updated_at("2026-03-01T11:10:00+00:00") + .add_message( + "u2", + role="user", + text="Confirm the surfaces agree.", + timestamp="2026-03-01T11:00:00+00:00", + ) + .save() + ) + + +# Provenance stamps a payload with the moment it was projected, so two reads of +# the same rows differ in these keys alone. Parity is about the rows and the +# envelope, so they are normalized rather than compared. +_PER_CALL_STAMPS = frozenset({"materialized_at", "generated_at", "checked_at"}) + + +def _stable(value: object) -> object: + """Return ``value`` with per-call generation stamps normalized.""" + if isinstance(value, Mapping): + return {key: ("" if key in _PER_CALL_STAMPS else _stable(item)) for key, item in value.items()} + if isinstance(value, list): + return [_stable(item) for item in value] + return value + + +def _canonical(payload: Mapping[str, object]) -> str: + return json.dumps(_stable(payload), sort_keys=True) + + +def _cli_payload(insight_type: InsightType) -> Mapping[str, object] | None: + """Read one insight list through the CLI's JSON surface.""" + result = CliRunner().invoke( + cli, + ["analyze", "insights", insight_type.resolved_cli_command_name, "--format", "json"], + catch_exceptions=False, + ) + if result.exit_code != 0: + return None + return extract_json_result(result.output) + + +@pytest.mark.asyncio +async def test_cli_mcp_and_api_insight_lists_are_identical(cli_workspace: dict[str, Path]) -> None: + db_path = cli_workspace["db_path"] + _seed(db_path) + archive = Polylogue(archive_root=cli_workspace["archive_root"], db_path=db_path) + try: + await archive.rebuild_insights() + + compared: list[str] = [] + populated: list[str] = [] + for insight_type in _parity_insight_types(): + api_items = await fetch_insights_async(insight_type, archive) + api_payload = insight_items_payload(api_items, insight_type) + + # The MCP tool derives its own (limit, offset) defaults from the + # registry; routing them back through the same product route is what + # makes the surfaces one surface. + spec = InsightListToolSpec.from_insight_type(insight_type) + default_limit = insight_type.mcp_default_limit + + def _clamp(value: object, fallback: int = default_limit) -> int: + return int(value) if isinstance(value, int) else fallback + + mcp_kwargs = spec.normalize_kwargs( + _clamp, + {name: default for name, default in spec.signature.kwdefaults.items() if name in {"limit", "offset"}}, + ) + mcp_items = await fetch_insights_async(insight_type, archive, **mcp_kwargs) + mcp_payload = insight_items_payload(mcp_items, insight_type) + + cli_payload = _cli_payload(insight_type) + + assert _canonical(mcp_payload) == _canonical(api_payload), ( + f"{insight_type.name}: MCP and API insight lists diverged" + ) + if cli_payload is not None: + assert _canonical(cli_payload) == _canonical(api_payload), ( + f"{insight_type.name}: CLI and API insight lists diverged" + ) + compared.append(insight_type.name) + if api_payload["total"]: + populated.append(insight_type.name) + + assert compared, "no insight type reached all three surfaces; the parity check compared nothing" + assert populated, "the seeded archive produced no insight rows; the payload comparison is vacuous" + finally: + await archive.close() diff --git a/tests/unit/core/test_readiness_capability.py b/tests/unit/core/test_readiness_capability.py index 158bae0f33..f017f369d6 100644 --- a/tests/unit/core/test_readiness_capability.py +++ b/tests/unit/core/test_readiness_capability.py @@ -689,7 +689,8 @@ def test_insight_entry_operation_and_catchup_adapters() -> None: SimpleNamespace( insight_name="session_profiles", display_name="Session Profiles", - verdict="incompatible", + table_present=True, + incompatible_count=3, row_count=3, repair_command="polylogue ops maintenance repair session-insights", evidence=("session_insight_status",), diff --git a/tests/unit/daemon/test_derived_stage_debt_surface.py b/tests/unit/daemon/test_derived_stage_debt_surface.py new file mode 100644 index 0000000000..ba188df88b --- /dev/null +++ b/tests/unit/daemon/test_derived_stage_debt_surface.py @@ -0,0 +1,127 @@ +"""A stalled derived stage is ordinary, typed, retryable convergence debt. + +Derived session rows have no lifecycle of their own. When their stage cannot +complete, the only thing that must happen is what happens for every other +derived object: a typed convergence-debt row, retryable, visible on the debt +surface, and reflected in the one readiness signal. + +Anti-vacuity: each assertion names a distinct link in that chain, so the test +goes red if the converger stops reporting the stalled stage, if the debt row +loses its retry schedule or its stage label, if ``archive_debt_list`` stops +projecting convergence rows, or if the insight coverage report stops reading +convergence debt and goes back to declaring readiness on its own. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from polylogue.api import Polylogue +from polylogue.daemon.convergence import ConvergenceStage, DaemonConverger +from polylogue.daemon.convergence_stages import make_fts_stage +from polylogue.operations.archive_debt import archive_debt_list +from polylogue.sources.live.convergence_debt import convergence_debt_from_states +from polylogue.sources.live.convergence_outcome import record_convergence_outcome +from polylogue.sources.live.cursor import CursorStore +from tests.infra.convergence_harness import ( + build_converged_archive, + debt_ledger_row, + rich_convergence_pathology, +) + +_STALL_ERROR = "derived stage stalled for the debt-surface fixture" + + +def _stalled_derived_stage() -> ConvergenceStage: + """The real stage contract with an execute that cannot complete.""" + + def _raise(_target: object) -> bool: + raise RuntimeError(_STALL_ERROR) + + return ConvergenceStage( + name="derived", + description="Refresh session-derived tables for new sessions", + check=lambda _path: True, + execute=_raise, + check_sessions=lambda session_ids: set(session_ids), + execute_sessions=_raise, + ) + + +def test_stalled_derived_stage_surfaces_as_retryable_convergence_debt(tmp_path: Path) -> None: + archive = build_converged_archive(tmp_path / "archive", rich_convergence_pathology()) + index_db = archive.root / "index.db" + ops_db = archive.root / "ops.db" + + converger = DaemonConverger((make_fts_stage(index_db), _stalled_derived_stage())) + states, _timings = converger.converge_batch(archive.source_paths) + + # The converger must report the stall rather than swallowing it. + assert any(not state.converged for state in states.values()) + + debts = convergence_debt_from_states(archive.source_paths, states) + assert {debt.stage for debt in debts} == {"derived"}, "the stall must be attributed to the derived stage" + + cursor = CursorStore(index_db, ops_db_path=ops_db) + for path in archive.source_paths: + record_convergence_outcome( + cursor, + path, + [debt for debt in debts if debt.path == path], + archive_root=archive.root, + ) + + # Typed and retryable in the durable ledger. + rows = [ + row + for path in archive.source_paths + if (row := debt_ledger_row(ops_db, stage="derived", subject_type="source_path", subject_id=str(path))) + is not None + ] + session_rows = [ + row + for session_id in archive.session_ids + if (row := debt_ledger_row(ops_db, stage="derived", subject_type="session_id", subject_id=session_id)) + is not None + ] + ledger_rows = rows + session_rows + assert ledger_rows, "the stalled derived stage recorded no convergence debt" + for row in ledger_rows: + assert row.status == "failed", "a stage that raised is a failure, not a deferral" + assert row.next_retry_at is not None, "convergence debt must carry a retry schedule" + assert row.last_error is not None and _STALL_ERROR in row.last_error + + # Projected on the operator debt surface, with the stage named. + payload = archive_debt_list(archive_root=archive.root, kinds=["convergence"]) + convergence_rows = [row for row in payload.rows if row.kind == "convergence"] + assert convergence_rows, "archive_debt_list reported no convergence debt for a stalled stage" + assert any(row.stage == "derived" for row in convergence_rows) + + +@pytest.mark.asyncio +async def test_insight_readiness_reads_convergence_debt_as_its_only_signal(tmp_path: Path) -> None: + archive = build_converged_archive(tmp_path / "archive", rich_convergence_pathology()) + index_db = archive.root / "index.db" + ops_db = archive.root / "ops.db" + + polylogue = Polylogue(archive_root=archive.root, db_path=index_db) + try: + converged = await polylogue.insight_readiness_report() + assert converged.converged is True, "a converged archive must report caught-up convergence" + assert converged.debt_stages == () + + CursorStore(index_db, ops_db_path=ops_db).record_convergence_debt( + stage="derived", + subject_type="session_id", + subject_id=archive.session_ids[0], + error=_STALL_ERROR, + ) + + stalled = await polylogue.insight_readiness_report() + finally: + await polylogue.close() + + assert stalled.converged is False, "readiness must follow convergence debt, not a private verdict" + assert "derived" in stalled.debt_stages diff --git a/tests/unit/insights/test_fallback_markers.py b/tests/unit/insights/test_fallback_markers.py index f8fe869ecb..9ae2f4cbe1 100644 --- a/tests/unit/insights/test_fallback_markers.py +++ b/tests/unit/insights/test_fallback_markers.py @@ -139,7 +139,6 @@ async def test_readiness_report_classifies_fallback_rows_as_degraded( ) profile = next(entry for entry in report.insights if entry.insight_name == "session_profiles") - assert profile.verdict == "degraded" assert profile.degraded_count == 1 # The seeded session materializes weak work-events and tool-less # phases; the taxonomy surfaces those reasons explicitly. @@ -148,6 +147,6 @@ async def test_readiness_report_classifies_fallback_rows_as_degraded( assert any("degraded=1" in line for line in profile.evidence) assert any("fallback_reason=" in line for line in profile.evidence) - # Aggregate verdict promotes the worst entry; here both profiles and - # enrichments are degraded but nothing is incompatible/stale/partial. - assert report.aggregate_verdict == "degraded" + # Fallback markers describe row quality, not divergence: the rows still + # reflect the sources they were built from. + assert not profile.diverged From 9bec5f3d905d3934b72fdfc1ed6053b6172b0dee Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 01:50:13 +0200 Subject: [PATCH 30/47] fix: gate session-insight status counts on the relations they read Status counts read product tables without requiring them to exist. The reads were unreachable behind the insight_materialization KeyError; removing that exposed them, so an archive whose derived tables are not built yet raises "no such table" instead of reporting zero. Four count descriptors read a product table they did not gate on. Two relations are query-time views over session_profiles and session_work_events: sqlite_master lists a view whether or not its body's tables exist, so presence alone never made them readable. The view dependency cannot be read off a descriptor's query text, so it is declared once and every gate expands through it. A descriptor now gates on the set of relations its query reads, and an import-time check refuses any descriptor whose query names a product table it does not gate on. Also repoint two test references at storage.derived, renamed from storage.insights in #4614; the stale import failed collection on master. Co-Authored-By: Claude Opus 5 --- polylogue/storage/derived/session/rebuild.py | 2 +- polylogue/storage/derived/session/status.py | 82 +++++++++++++------ tests/unit/cli/test_insights.py | 2 +- .../unit/cli/test_insights_command_runtime.py | 2 +- tests/unit/storage/test_derived_status.py | 2 +- 5 files changed, 63 insertions(+), 27 deletions(-) diff --git a/polylogue/storage/derived/session/rebuild.py b/polylogue/storage/derived/session/rebuild.py index 4e8b7fdc35..7643d9b6a7 100644 --- a/polylogue/storage/derived/session/rebuild.py +++ b/polylogue/storage/derived/session/rebuild.py @@ -962,7 +962,7 @@ def compute_session_insight_bundles( Each job is expected to be pure Python compute over an already-hydrated ``Session`` (see ``build_session_insight_record_bundles`` and - ``storage/insights/session/refresh.py``): no SQLite connection, no + ``storage/derived/session/refresh.py``): no SQLite connection, no shared mutable state, so results are safe to compute concurrently. A caller-supplied ``stage_timing_add`` sink is the one shared piece of mutable state jobs may still touch; callers that pass one MUST make its diff --git a/polylogue/storage/derived/session/status.py b/polylogue/storage/derived/session/status.py index 8fdfe10f0b..2c4b08ab5c 100644 --- a/polylogue/storage/derived/session/status.py +++ b/polylogue/storage/derived/session/status.py @@ -3,7 +3,9 @@ from __future__ import annotations import dataclasses +import re import sqlite3 +from collections.abc import Sequence from dataclasses import dataclass from typing import TypeAlias @@ -26,6 +28,24 @@ TablePresence: TypeAlias = dict[str, bool] StatusCounts: TypeAlias = dict[str, int] +# Query-time relations select through these tables. A view is listed in +# sqlite_master whether or not the tables in its body exist, so presence alone +# does not make it readable: selecting from it raises "no such table" until its +# sources are built. The dependency lives in the view body and cannot be read +# off a descriptor's query text, so it is declared once here and every gate +# expands through it. +_VIEW_DEPENDENCIES: dict[str, tuple[str, ...]] = { + "threads": ("session_profiles", "session_work_events"), + "session_tag_rollups": ("session_profiles",), +} + + +def _relations_readable(tables: TablePresence, keys: Sequence[str]) -> bool: + """Whether every named relation, and everything it selects through, exists.""" + return all( + tables[key] and all(tables[dependency] for dependency in _VIEW_DEPENDENCIES.get(key, ())) for key in keys + ) + @dataclass(frozen=True) class SessionInsightTableDescriptor: @@ -40,17 +60,20 @@ class SessionInsightTableDescriptor: def exists_sql(self) -> str: return f"SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name='{self.table_name}'" + def _readable(self, tables: TablePresence) -> bool: + return _relations_readable(tables, (self.key,)) + def count_sync(self, conn: sqlite3.Connection, tables: TablePresence) -> tuple[str, int] | None: if self.count_key is None: return None - if not tables[self.key]: + if not self._readable(tables): return (self.count_key, 0) return (self.count_key, _count_sync(conn, self.count_sql or f"SELECT COUNT(*) FROM {self.table_name}")) async def count_async(self, conn: aiosqlite.Connection, tables: TablePresence) -> tuple[str, int] | None: if self.count_key is None: return None - if not tables[self.key]: + if not self._readable(tables): return (self.count_key, 0) return (self.count_key, await _count_async(conn, self.count_sql or f"SELECT COUNT(*) FROM {self.table_name}")) @@ -112,7 +135,7 @@ class SessionInsightCountDescriptor: count_key: str sql: str - table_key: str | None = None + table_keys: tuple[str, ...] = () params: tuple[object, ...] = () requires_freshness: bool = False fallback_count_key: str | None = None @@ -121,7 +144,7 @@ class SessionInsightCountDescriptor: def _should_query(self, tables: TablePresence, *, verify_freshness: bool) -> bool: if self.requires_freshness and not verify_freshness: return False - return self.table_key is None or tables[self.table_key] + return _relations_readable(tables, self.table_keys) def _fallback(self, counts: StatusCounts) -> int: if self.fallback_count_key is not None: @@ -446,93 +469,96 @@ async def _stale_session_profile_count_sql_async(conn: aiosqlite.Connection) -> _COUNT_DESCRIPTORS: tuple[SessionInsightCountDescriptor, ...] = ( SessionInsightCountDescriptor( count_key="missing_profile_row_count", - table_key="session_profiles", + table_keys=("session_profiles",), sql=MISSING_SESSION_PROFILE_COUNT_SQL, fallback_count_key="total_sessions", ), SessionInsightCountDescriptor( count_key="orphan_profile_row_count", - table_key="session_profiles", + table_keys=("session_profiles",), sql=ORPHAN_SESSION_PROFILE_COUNT_SQL, requires_freshness=True, ), SessionInsightCountDescriptor( count_key="missing_latency_profile_row_count", - table_key="session_latency_profiles", + table_keys=("session_latency_profiles", "session_profiles"), sql=MISSING_SESSION_LATENCY_PROFILE_COUNT_SQL, fallback_count_key="profile_row_count", ), SessionInsightCountDescriptor( count_key="stale_latency_profile_row_count", - table_key="session_latency_profiles", + table_keys=("session_latency_profiles",), sql=STALE_SESSION_LATENCY_PROFILE_COUNT_SQL, params=(SESSION_INSIGHT_MATERIALIZER_VERSION,), requires_freshness=True, ), SessionInsightCountDescriptor( count_key="orphan_latency_profile_row_count", - table_key="session_latency_profiles", + table_keys=("session_latency_profiles",), sql=ORPHAN_SESSION_LATENCY_PROFILE_COUNT_SQL, requires_freshness=True, ), SessionInsightCountDescriptor( count_key="expected_work_event_inference_count", - table_key="session_profiles", + table_keys=("session_profiles",), sql=EXPECTED_WORK_EVENT_COUNT_SQL, ), SessionInsightCountDescriptor( count_key="expected_phase_inference_count", - table_key="session_profiles", + table_keys=("session_profiles",), sql=EXPECTED_PHASE_COUNT_SQL, ), SessionInsightCountDescriptor( count_key="orphan_work_event_inference_count", - table_key="session_work_events", + table_keys=("session_work_events",), sql=ORPHAN_SESSION_WORK_EVENT_COUNT_SQL, requires_freshness=True, ), SessionInsightCountDescriptor( count_key="orphan_phase_inference_count", - table_key="session_phases", + table_keys=("session_phases",), sql=ORPHAN_SESSION_PHASE_COUNT_SQL, requires_freshness=True, ), SessionInsightCountDescriptor( count_key="stale_thread_count", - table_key="session_profiles", + table_keys=("session_profiles",), sql=STALE_THREAD_COUNT_SQL, params=(SESSION_INSIGHT_MATERIALIZER_VERSION,), requires_freshness=True, ), SessionInsightCountDescriptor( count_key="orphan_thread_count", - table_key="threads", + table_keys=("threads",), sql=ORPHAN_THREAD_COUNT_SQL, requires_freshness=True, ), SessionInsightCountDescriptor( count_key="expected_tag_rollup_count", - table_key="session_profiles", + table_keys=("session_profiles",), sql=EXPECTED_SESSION_TAG_ROLLUP_COUNT_SQL, requires_freshness=True, fallback_count_key="tag_rollup_count", ), SessionInsightCountDescriptor( count_key="stale_tag_rollup_count", - table_key="session_tag_rollups", + table_keys=("session_tag_rollups",), sql="SELECT COUNT(*) FROM session_tag_rollups WHERE materialized_at != 'query-time'", requires_freshness=True, ), ) # The status counts are splatted into ``SessionInsightStatusSnapshot``, and -# descriptors index ``tables``/``counts`` by name. Every emitted key must -# therefore be a snapshot field, every ``table_key`` a table the presence probe -# reports, and every referenced fallback/source key one some descriptor emits. -# A key that satisfies none of these raises only when a status call reaches it, -# so all three are checked at import. +# descriptors index ``tables``/``counts`` by name. Every emitted key must be a +# snapshot field, every referenced fallback/source key one some descriptor +# emits, and every product table a query reads must be one the descriptor gates +# on -- an ungated read raises "no such table" on an archive whose derived +# tables have not been built yet. Each of these raises only when a status call +# reaches it, so all of them are checked at import. _SNAPSHOT_COUNT_FIELDS = frozenset(field.name for field in dataclasses.fields(SessionInsightStatusSnapshot)) _TABLE_PRESENCE_KEYS = frozenset(descriptor.key for descriptor in _TABLE_DESCRIPTORS) +# ``sessions`` is the archive's own spine and is never absent where status runs. +_PRODUCT_TABLES = frozenset(descriptor.table_name for descriptor in _TABLE_DESCRIPTORS) - {"sessions"} _EMITTED_COUNT_KEYS = ( {"total_sessions", "root_threads"} | {descriptor.count_key for descriptor in _TABLE_DESCRIPTORS if descriptor.count_key is not None} @@ -546,8 +572,10 @@ async def _stale_session_profile_count_sql_async(conn: aiosqlite.Connection) -> if _unknown_tables := sorted( ( - {descriptor.table_key for descriptor in _COUNT_DESCRIPTORS if descriptor.table_key is not None} + {table_key for descriptor in _COUNT_DESCRIPTORS for table_key in descriptor.table_keys} | {descriptor.table_key for descriptor in _FTS_DESCRIPTORS} + | set(_VIEW_DEPENDENCIES) + | {table for tables in _VIEW_DEPENDENCIES.values() for table in tables} ) - _TABLE_PRESENCE_KEYS ): @@ -562,6 +590,14 @@ async def _stale_session_profile_count_sql_async(conn: aiosqlite.Connection) -> ): raise RuntimeError(f"status descriptors reference counts nothing emits: {_unknown_references}") +if _ungated_reads := sorted( + (descriptor.count_key, table) + for descriptor in _COUNT_DESCRIPTORS + for table in _PRODUCT_TABLES + if re.search(rf"\b{re.escape(table)}\b", descriptor.sql) and table not in descriptor.table_keys +): + raise RuntimeError(f"status descriptors read product tables they do not gate on: {_ungated_reads}") + def _to_int(row: tuple[object, ...] | sqlite3.Row | None) -> int: if not row: diff --git a/tests/unit/cli/test_insights.py b/tests/unit/cli/test_insights.py index 6bc4f67dde..99b18c2cb5 100644 --- a/tests/unit/cli/test_insights.py +++ b/tests/unit/cli/test_insights.py @@ -588,7 +588,7 @@ def test_insights_status_plain(cli_workspace: CliWorkspace) -> None: result = runner.invoke(cli, ["ops", "insights", "status", "--insight", "profiles"], catch_exceptions=False) assert result.exit_code == 0 - assert "Insight Readiness: degraded" in result.output + assert "Convergence: caught up" in result.output assert "session_profiles: degraded" in result.output diff --git a/tests/unit/cli/test_insights_command_runtime.py b/tests/unit/cli/test_insights_command_runtime.py index 1ab5c327a4..c2c2e5385a 100644 --- a/tests/unit/cli/test_insights_command_runtime.py +++ b/tests/unit/cli/test_insights_command_runtime.py @@ -192,7 +192,7 @@ def test_render_status_plain_and_export_plain_cover_optional_sections( insights_module._render_export_plain(_export_result(tmp_path)) output = capsys.readouterr().out - assert "Insight Readiness: partial" in output + assert "Convergence: debt in derived" in output assert "Scope: origin=codex-session since=2026-04-01 until=2026-04-30" in output assert "session_profiles: partial rows=7 expected=10" in output assert "missing=1 stale=2 orphan=3 incompatible=4" in output diff --git a/tests/unit/storage/test_derived_status.py b/tests/unit/storage/test_derived_status.py index f4f946e920..c184f1b27a 100644 --- a/tests/unit/storage/test_derived_status.py +++ b/tests/unit/storage/test_derived_status.py @@ -80,7 +80,7 @@ def test_collect_derived_statuses_preserves_independent_readiness_conditions( monkeypatch: pytest.MonkeyPatch, ) -> None: from polylogue.storage.derived import derived_status as derived_status_mod - from polylogue.storage.insights.session.runtime import SessionInsightStatusSnapshot + from polylogue.storage.derived.session.runtime import SessionInsightStatusSnapshot conn = sqlite3.connect(":memory:") snapshot = SessionInsightStatusSnapshot( From 8694bd193c52787bc9d0c17cc3951d8a2b4fbf8b Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 02:11:48 +0200 Subject: [PATCH 31/47] test(cli): update planner statistics snapshot --- tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr b/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr index bee143a9d0..2d3c67e62a 100644 --- a/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr +++ b/tests/unit/cli/__snapshots__/test_plain_cli_snapshots.ambr @@ -1158,14 +1158,14 @@ "path": "", "exists": true, "wal_bytes": 0, - "sqlite_stat1_rows": 15, + "sqlite_stat1_rows": 14, "planner_stats_present": true }, "index": { "path": "", "exists": true, "wal_bytes": 0, - "sqlite_stat1_rows": 38, + "sqlite_stat1_rows": 37, "planner_stats_present": true }, "embeddings": { From 66b4d3ca40961e1829289393886def6ebb827c08 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 02:26:54 +0200 Subject: [PATCH 32/47] test(storage): follow multi-relation status descriptors --- tests/unit/storage/test_session_insight_status_descriptors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/storage/test_session_insight_status_descriptors.py b/tests/unit/storage/test_session_insight_status_descriptors.py index 5d3df47cd9..547b127cd6 100644 --- a/tests/unit/storage/test_session_insight_status_descriptors.py +++ b/tests/unit/storage/test_session_insight_status_descriptors.py @@ -89,7 +89,7 @@ async def test_fts_descriptor_async_can_skip_distinct_freshness_counts(tmp_path: def test_count_descriptor_uses_fallback_when_freshness_is_disabled() -> None: descriptor = SessionInsightCountDescriptor( count_key="expected_rows", - table_key="source_table", + table_keys=("source_table",), sql="SELECT 99", requires_freshness=True, fallback_count_key="materialized_rows", @@ -390,7 +390,7 @@ def test_status_descriptors_resolve_against_declared_tables_and_snapshot_fields( emitted |= {descriptor.duplicate_count_key for descriptor in _FTS_DESCRIPTORS} referenced = {descriptor.table_key for descriptor in _FTS_DESCRIPTORS} - referenced |= {descriptor.table_key for descriptor in _COUNT_DESCRIPTORS if descriptor.table_key is not None} + referenced |= {table_key for descriptor in _COUNT_DESCRIPTORS for table_key in descriptor.table_keys} assert referenced <= declared_tables assert emitted <= snapshot_fields From 381f88612698e0de202d26859633d3522d4314f8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 02:30:36 +0200 Subject: [PATCH 33/47] fix(status): surface convergence ledger read failures --- .../storage/sqlite/archive_tiers/archive.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/archive.py b/polylogue/storage/sqlite/archive_tiers/archive.py index 9e0274ea1f..6be1512d09 100644 --- a/polylogue/storage/sqlite/archive_tiers/archive.py +++ b/polylogue/storage/sqlite/archive_tiers/archive.py @@ -5425,26 +5425,23 @@ def _derived_convergence_signal(self) -> tuple[bool | None, tuple[str, ...]]: """Report whether convergence has caught up, and which stages have not. Derived rows have no lifecycle of their own: their readiness is exactly - the ordinary convergence signal. ops.db is disposable, so a ledger that - cannot be read reports ``None`` -- unknown, never converged. + the ordinary convergence signal. A missing ledger reports ``None``; + read failures remain visible to the status error boundary. """ if not self.ops_db_path.exists(): return (None, ()) + conn = sqlite3.connect(f"file:{self.ops_db_path}?mode=ro", uri=True) try: - conn = sqlite3.connect(f"file:{self.ops_db_path}?mode=ro", uri=True) - try: - present = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'convergence_debt'" - ).fetchone() - if present is None: - return (None, ()) - rows = conn.execute( - "SELECT DISTINCT stage FROM convergence_debt WHERE status IN ('failed', 'deferred') ORDER BY stage" - ).fetchall() - finally: - conn.close() - except sqlite3.Error: - return (None, ()) + present = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'convergence_debt'" + ).fetchone() + if present is None: + return (None, ()) + rows = conn.execute( + "SELECT DISTINCT stage FROM convergence_debt WHERE status IN ('failed', 'deferred') ORDER BY stage" + ).fetchall() + finally: + conn.close() stages = tuple(str(row[0]) for row in rows) return (not stages, stages) From 1655f6645f92ed9b365ebc6fed79b0c7401939f0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 19:16:08 +0200 Subject: [PATCH 34/47] feat: Compile and consume the physical blob disposition plan Give every physical blob exactly one disposition proven against a configured source, restore sole-copy carriers into their ordinary spool, and delete only proven-redundant unreferenced objects through the canonical blob-GC seam. Hook-event and browser-capture carriers are proven by the owning production read route rather than by bytes: acquisition derives fields the spool file does not carry, so byte equality reports reproducible material as a sole copy. Co-Authored-By: Claude Opus 5 --- docs/maintenance.md | 38 + .../cli/commands/maintenance/__init__.py | 8 +- .../commands/maintenance/_blob_disposition.py | 254 ++++++ polylogue/maintenance/blob_disposition.py | 849 ++++++++++++++++++ .../maintenance/blob_disposition_apply.py | 475 ++++++++++ polylogue/sources/hooks.py | 13 + .../test_blob_disposition_apply.py | 426 +++++++++ .../maintenance/test_blob_disposition_plan.py | 338 +++++++ 8 files changed, 2400 insertions(+), 1 deletion(-) create mode 100644 polylogue/cli/commands/maintenance/_blob_disposition.py create mode 100644 polylogue/maintenance/blob_disposition.py create mode 100644 polylogue/maintenance/blob_disposition_apply.py create mode 100644 tests/unit/maintenance/test_blob_disposition_apply.py create mode 100644 tests/unit/maintenance/test_blob_disposition_plan.py diff --git a/docs/maintenance.md b/docs/maintenance.md index c096b8be3f..7cbe450fde 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -295,6 +295,44 @@ recorded separately in the receipt so a retry can safely continue an additive repair. Reindex acceptance runs the same closure check against the candidate index before promotion. +### `polylogue ops maintenance blob-disposition` — physical namespace disposition + +One-time transition tooling for the blob-store maneuver. `plan` is read-only: +it walks the complete physical namespace and gives every object exactly one +disposition proven against a configured source — `source_present`, +`superseded_prefix`, `restore_required`, or `unresolved`. A plan is acceptable +only at zero unresolved members, and its digest binds the archive identity, +the namespace, the denominators, and every member outcome. + +```bash +polylogue ops maintenance blob-disposition plan \ + --archive-root /path/to/archive \ + --output /path/to/disposition-plan.json --output-format json +polylogue ops maintenance blob-disposition apply \ + --archive-root /path/to/archive \ + --plan /path/to/disposition-plan.json \ + --authorized-digest \ + --receipt /path/to/new/disposition-receipt.json --active +``` + +`restore` is the additive half on its own: it publishes sole-copy carriers +into their ordinary spool and deletes nothing, so it does not wait on the +plan reaching zero unresolved. `apply` is a dry rehearsal without `--active`. It makes no classification +judgment: it revalidates every member's own proof immediately before its +effect, restores sole-copy carriers into their ordinary spool before any +deletion, never touches a historical carrier during restoration, and deletes +only unreferenced members through the canonical blob-GC seam. Any drift — a +changed source, a changed object, a new referent, a different digest or +denominator — refuses the whole plan. + +Hook-event and browser-capture carriers are proven by the owning production +read route, not by bytes: acquisition derives fields the spool file does not +carry, so byte equality would misreport reproducible material as a sole copy. + +Deletion trigger: this command, both maintenance modules, and their tests are +removed with the terminal disposition receipt. The recurring liveness, +publication, GC, and spool-admission laws stay with their owners. + ### `polylogue ops maintenance preview` — staleness inventory Read-only. Produces a per-model inventory of stale, missing, orphan, diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 79b0fee903..17e77c795c 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -219,9 +219,15 @@ "blob_residue_compare_command", "Compare present blob-residue candidates through the production parse route.", ), + ( + "blob-disposition", + "_blob_disposition", + "blob_disposition_group", + "Compile or consume the physical blob namespace disposition plan.", + ), ) -_NESTED_GROUP_COMMANDS = frozenset({"archive-root-relocation", "source-continuity-recovery"}) +_NESTED_GROUP_COMMANDS = frozenset({"archive-root-relocation", "blob-disposition", "source-continuity-recovery"}) @click.group("maintenance") diff --git a/polylogue/cli/commands/maintenance/_blob_disposition.py b/polylogue/cli/commands/maintenance/_blob_disposition.py new file mode 100644 index 0000000000..e3297b3df7 --- /dev/null +++ b/polylogue/cli/commands/maintenance/_blob_disposition.py @@ -0,0 +1,254 @@ +"""``maintenance blob-disposition``: compile or consume one disposition plan.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import click + + +@click.group("blob-disposition") +def blob_disposition_group() -> None: + """Plan and consume the physical blob namespace disposition.""" + + +@blob_disposition_group.command("plan") +@click.option( + "--archive-root", + type=click.Path(path_type=Path, exists=True, file_okay=False, readable=True), + required=True, + help="Archive root whose blob namespace is planned.", +) +@click.option( + "--output", + type=click.Path(path_type=Path, dir_okay=False), + required=True, + help="Destination for the immutable plan artifact.", +) +@click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) +def blob_disposition_plan_command(archive_root: Path, output: Path, output_format: str) -> None: + """Compile a read-only, zero-unknown disposition plan. Never mutates.""" + from polylogue.maintenance.blob_disposition import compile_disposition_plan, resolve_disposition_roots + + _, hook_sources, capture_spool = resolve_disposition_roots(archive_root) + try: + plan = compile_disposition_plan( + archive_root=archive_root, + blob_root=archive_root / "blob", + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(plan.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + "\n") + except (OSError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + summary = { + "plan": str(output), + "digest": plan.digest(), + "accepted": plan.accepted, + "counts": plan.counts, + "bytes_by_disposition": plan.bytes_by_disposition, + "denominator": plan.denominator.to_dict(), + } + if output_format == "json": + click.echo(json.dumps(summary, sort_keys=True)) + return + click.echo(f"Disposition plan: {output}") + click.echo(f"Digest: {plan.digest()}") + click.echo(f"Accepted (zero unresolved): {plan.accepted}") + click.echo(f"Counts: {json.dumps(plan.counts, sort_keys=True)}") + click.echo("Read-only: true") + + +@blob_disposition_group.command("restore") +@click.option( + "--archive-root", + type=click.Path(path_type=Path, exists=True, file_okay=False, readable=True), + required=True, + help="Archive root the plan was compiled from.", +) +@click.option( + "--plan", + "plan_path", + type=click.Path(path_type=Path, exists=True, dir_okay=False, readable=True), + required=True, + help="The plan naming the sole-copy carriers to restore.", +) +@click.option("--authorized-digest", required=True, help="Digest of the reviewed plan.") +@click.option( + "--receipt", + type=click.Path(path_type=Path, dir_okay=False), + required=True, + help="Destination for the restoration receipt.", +) +@click.option( + "--active", + is_flag=True, + default=False, + help="Perform the restorations. Without it the run is a dry rehearsal.", +) +@click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) +def blob_disposition_restore_command( + archive_root: Path, + plan_path: Path, + authorized_digest: str, + receipt: Path, + active: bool, + output_format: str, +) -> None: + """Restore sole-copy carriers into their ordinary spool. Deletes nothing. + + Restoration is additive, so it does not wait on the whole plan reaching + zero unresolved: withholding it would leave the only carrier of wanted + material unpreserved while unrelated objects are still being classified. + """ + from polylogue.maintenance.blob_disposition import ( + BlobDispositionPlan, + build_disposition_context, + resolve_disposition_roots, + ) + from polylogue.maintenance.blob_disposition_apply import ( + TOOL_VERSION, + DispositionApplyReceipt, + restore_plan_members, + write_receipt, + ) + + hooks_root, hook_sources, capture_spool = resolve_disposition_roots(archive_root) + try: + plan = BlobDispositionPlan.from_dict(json.loads(plan_path.read_text(encoding="utf-8"))) + if plan.digest() != authorized_digest: + raise click.ClickException("authorized digest does not match the plan") + context = build_disposition_context( + archive_root=archive_root, + blob_root=archive_root / "blob", + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + results = restore_plan_members( + plan, + context=context, + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=not active, + ) + result = DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=not active, + results=results, + ) + write_receipt(receipt, result) + except (OSError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + if output_format == "json": + click.echo(json.dumps({"receipt": str(receipt), "ok": result.ok, "counts": result.counts}, sort_keys=True)) + else: + click.echo(f"Restoration receipt: {receipt}") + click.echo(f"Dry run: {not active}") + click.echo(f"Counts: {json.dumps(result.counts, sort_keys=True)}") + if not result.ok: + raise SystemExit(1) + + +@blob_disposition_group.command("apply") +@click.option( + "--archive-root", + type=click.Path(path_type=Path, exists=True, file_okay=False, readable=True), + required=True, + help="Archive root the authorized plan was compiled from.", +) +@click.option( + "--plan", + "plan_path", + type=click.Path(path_type=Path, exists=True, dir_okay=False, readable=True), + required=True, + help="The exact accepted plan artifact.", +) +@click.option("--authorized-digest", required=True, help="Digest of the independently accepted plan.") +@click.option( + "--receipt", + type=click.Path(path_type=Path, dir_okay=False), + required=True, + help="Destination for the complete before/after receipt.", +) +@click.option( + "--active", + is_flag=True, + default=False, + help="Perform the authorized effects. Without it the run is a dry rehearsal.", +) +@click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) +def blob_disposition_apply_command( + archive_root: Path, + plan_path: Path, + authorized_digest: str, + receipt: Path, + active: bool, + output_format: str, +) -> None: + """Restore sole copies, then delete proven-redundant objects.""" + from polylogue.config import Config + from polylogue.maintenance.blob_disposition import ( + BlobDispositionPlan, + build_disposition_context, + resolve_disposition_roots, + ) + from polylogue.maintenance.blob_disposition_apply import apply_disposition_plan, write_receipt + from polylogue.maintenance.offline_guard import offline_writer_block_reason + from polylogue.paths import render_root + + hooks_root, hook_sources, capture_spool = resolve_disposition_roots(archive_root) + try: + plan = BlobDispositionPlan.from_dict(json.loads(plan_path.read_text(encoding="utf-8"))) + context = build_disposition_context( + archive_root=archive_root, + blob_root=archive_root / "blob", + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + block_reason = offline_writer_block_reason( + Config(archive_root=archive_root, render_root=render_root(), sources=[]) + ) + result = apply_disposition_plan( + plan, + context=context, + authorized_digest=authorized_digest, + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + writer_block_reason=block_reason, + dry_run=not active, + ) + write_receipt(receipt, result) + except (OSError, RuntimeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + summary = {"receipt": str(receipt), "ok": result.ok, "counts": result.counts, "blockers": list(result.blockers)} + if output_format == "json": + click.echo(json.dumps(summary, sort_keys=True)) + else: + click.echo(f"Disposition receipt: {receipt}") + click.echo(f"Dry run: {not active}") + click.echo(f"Counts: {json.dumps(result.counts, sort_keys=True)}") + for blocker in result.blockers: + click.echo(f"Blocked: {blocker}") + if not result.ok: + raise SystemExit(1) + + +__all__ = [ + "blob_disposition_apply_command", + "blob_disposition_group", + "blob_disposition_plan_command", + "blob_disposition_restore_command", +] diff --git a/polylogue/maintenance/blob_disposition.py b/polylogue/maintenance/blob_disposition.py new file mode 100644 index 0000000000..446ba22a6f --- /dev/null +++ b/polylogue/maintenance/blob_disposition.py @@ -0,0 +1,849 @@ +"""Read-only disposition plan for the physical blob namespace. + +The blob store is forensic evidence, never desired-state authority. Every +physical object therefore receives exactly one disposition proven against a +configured source: + +``source_present`` + Current source material reproduces the object's content, byte-identically + or through the owning production route's semantic equality. The object is + redundant storage and may be removed. +``superseded_prefix`` + The object is the exact prefix of a larger retained carrier of the same + logical source item (append lineage). It may be removed. +``restore_required`` + The object is the only verified carrier of wanted material and names an + ordinary spool destination that current acquisition admits. Restoration + precedes any removal. +``unresolved`` + Nothing above holds. Unresolved blocks: it is never downgraded to + discard, and it never authorizes restoration. + +A plan is acceptable only at zero unresolved members. It is immutable, bound +to the archive identity, blob namespace identity, and exact denominators it +was compiled from, and consumed by :mod:`polylogue.maintenance. +blob_disposition_apply` under a separate authorization. + +This is a one-time transition planner. Its deletion trigger is the terminal +disposition receipt: once the physical namespace is accounted for, this +module and its apply sibling go with it, and only the recurring liveness, +publication, GC, and spool-admission laws remain in their owners. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sqlite3 +from collections.abc import Mapping, Sequence +from contextlib import closing +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from typing import IO, Protocol + +from polylogue.storage.blob_store import BlobNamespaceEntry, BlobNamespaceEntryKind, BlobStore + +TOOL_VERSION = "blob-disposition-plan-v1" + +_HASH_CHUNK_BYTES = 1 << 20 +# An object larger than this is never one of the small JSON envelopes the +# spool provers own; probing it would read hundreds of megabytes to decide a +# question its size already answers. +_MAX_ENVELOPE_PROBE_BYTES = 256 << 20 + + +class BlobDispositionError(RuntimeError): + """Raised when a disposition plan cannot be compiled or trusted.""" + + +class BlobDisposition(StrEnum): + """The only terminal dispositions a physical blob may receive.""" + + SOURCE_PRESENT = "source_present" + SUPERSEDED_PREFIX = "superseded_prefix" + RESTORE_REQUIRED = "restore_required" + UNRESOLVED = "unresolved" + + +class SourceProofMode(StrEnum): + """How a prover established that current source material holds the content.""" + + BYTE_IDENTICAL = "byte_identical" + SEMANTIC_EQUIVALENT = "semantic_equivalent" + STRICT_PREFIX = "strict_prefix" + + +class RestorationDestination(StrEnum): + """Ordinary spool destinations current acquisition already admits.""" + + HOOK_EVENT_SPOOL = "hook_event_spool" + BROWSER_CAPTURE_SPOOL = "browser_capture_spool" + + +@dataclass(frozen=True, slots=True) +class SourceProof: + """One prover's evidence that a configured source holds the content.""" + + prover: str + mode: SourceProofMode + source_id: str + source_path: str + detail: str = "" + + def to_dict(self) -> dict[str, str]: + return { + "prover": self.prover, + "mode": self.mode.value, + "source_id": self.source_id, + "source_path": self.source_path, + "detail": self.detail, + } + + +@dataclass(frozen=True, slots=True) +class RestorationTarget: + """Where a sole-copy carrier is restored before its removal is considered.""" + + destination: RestorationDestination + logical_id: str + + def to_dict(self) -> dict[str, str]: + return {"destination": self.destination.value, "logical_id": self.logical_id} + + +@dataclass(frozen=True, slots=True) +class BlobDispositionMember: + """One physical blob and its single proven disposition.""" + + blob_hash: str + size_bytes: int + referenced: bool + disposition: BlobDisposition + reason: str + proof: SourceProof | None = None + restoration: RestorationTarget | None = None + + def to_dict(self) -> dict[str, object]: + return { + "blob_hash": self.blob_hash, + "size_bytes": self.size_bytes, + "referenced": self.referenced, + "disposition": self.disposition.value, + "reason": self.reason, + "proof": self.proof.to_dict() if self.proof is not None else None, + "restoration": self.restoration.to_dict() if self.restoration is not None else None, + } + + +@dataclass(frozen=True, slots=True) +class BlobDispositionDenominator: + """The exact population a plan was compiled from.""" + + physical_file_count: int + distinct_hash_count: int + total_bytes: int + referenced_hash_count: int + referenced_present_count: int + referenced_absent_count: int + invalid_namespace_entries: tuple[str, ...] = () + + def to_dict(self) -> dict[str, object]: + return { + "physical_file_count": self.physical_file_count, + "distinct_hash_count": self.distinct_hash_count, + "total_bytes": self.total_bytes, + "referenced_hash_count": self.referenced_hash_count, + "referenced_present_count": self.referenced_present_count, + "referenced_absent_count": self.referenced_absent_count, + "invalid_namespace_entries": list(self.invalid_namespace_entries), + } + + +@dataclass(frozen=True, slots=True) +class BlobDispositionPlan: + """An immutable, identity-bound, zero-unknown disposition plan.""" + + tool_version: str + archive_root: str + blob_root: str + denominator: BlobDispositionDenominator + members: tuple[BlobDispositionMember, ...] + + @property + def counts(self) -> dict[str, int]: + counts = {disposition.value: 0 for disposition in BlobDisposition} + for member in self.members: + counts[member.disposition.value] += 1 + return counts + + @property + def bytes_by_disposition(self) -> dict[str, int]: + totals = {disposition.value: 0 for disposition in BlobDisposition} + for member in self.members: + totals[member.disposition.value] += member.size_bytes + return totals + + @property + def unresolved_count(self) -> int: + return self.counts[BlobDisposition.UNRESOLVED.value] + + @property + def accepted(self) -> bool: + """A plan is acceptable only when nothing is unexplained.""" + return self.unresolved_count == 0 and not self.denominator.invalid_namespace_entries + + def members_for(self, disposition: BlobDisposition) -> tuple[BlobDispositionMember, ...]: + return tuple(member for member in self.members if member.disposition is disposition) + + def to_dict(self) -> dict[str, object]: + return { + "tool_version": self.tool_version, + "archive_root": self.archive_root, + "blob_root": self.blob_root, + "denominator": self.denominator.to_dict(), + "counts": self.counts, + "bytes_by_disposition": self.bytes_by_disposition, + "unresolved_count": self.unresolved_count, + "accepted": self.accepted, + "read_only": True, + "members": [member.to_dict() for member in self.members], + } + + def digest(self) -> str: + """Bind identity, denominators, and every exact member outcome.""" + payload = { + "tool_version": self.tool_version, + "archive_root": self.archive_root, + "blob_root": self.blob_root, + "denominator": self.denominator.to_dict(), + "members": [member.to_dict() for member in self.members], + } + canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + @classmethod + def from_dict(cls, payload: Mapping[str, object]) -> BlobDispositionPlan: + """Reload a persisted plan without re-deriving any judgment.""" + try: + denominator = payload["denominator"] + raw_members = payload["members"] + if not isinstance(denominator, Mapping) or not isinstance(raw_members, list): + raise BlobDispositionError("plan denominator and members must be structured") + members = tuple(_member_from_dict(item) for item in raw_members) + return cls( + tool_version=str(payload["tool_version"]), + archive_root=str(payload["archive_root"]), + blob_root=str(payload["blob_root"]), + denominator=BlobDispositionDenominator( + physical_file_count=int(denominator["physical_file_count"]), + distinct_hash_count=int(denominator["distinct_hash_count"]), + total_bytes=int(denominator["total_bytes"]), + referenced_hash_count=int(denominator["referenced_hash_count"]), + referenced_present_count=int(denominator["referenced_present_count"]), + referenced_absent_count=int(denominator["referenced_absent_count"]), + invalid_namespace_entries=tuple( + str(entry) for entry in denominator.get("invalid_namespace_entries", ()) + ), + ), + members=members, + ) + except (KeyError, TypeError, ValueError) as exc: + raise BlobDispositionError(f"unreadable disposition plan: {exc}") from exc + + +def _member_from_dict(payload: object) -> BlobDispositionMember: + if not isinstance(payload, Mapping): + raise BlobDispositionError("plan member must be an object") + proof_payload = payload.get("proof") + proof = None + if isinstance(proof_payload, Mapping): + proof = SourceProof( + prover=str(proof_payload["prover"]), + mode=SourceProofMode(str(proof_payload["mode"])), + source_id=str(proof_payload["source_id"]), + source_path=str(proof_payload["source_path"]), + detail=str(proof_payload.get("detail", "")), + ) + restoration_payload = payload.get("restoration") + restoration = None + if isinstance(restoration_payload, Mapping): + restoration = RestorationTarget( + destination=RestorationDestination(str(restoration_payload["destination"])), + logical_id=str(restoration_payload["logical_id"]), + ) + return BlobDispositionMember( + blob_hash=str(payload["blob_hash"]), + size_bytes=int(payload["size_bytes"]), + referenced=bool(payload["referenced"]), + disposition=BlobDisposition(str(payload["disposition"])), + reason=str(payload["reason"]), + proof=proof, + restoration=restoration, + ) + + +class BlobSourceProver(Protocol): + """Establishes that configured source material still holds a blob's content.""" + + name: str + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: ... + + +class BlobRestorationResolver(Protocol): + """Names the ordinary spool destination a sole-copy carrier belongs to.""" + + def restoration_target(self, path: Path) -> RestorationTarget | None: ... + + +def _read_envelope(path: Path, *, expected_keys: frozenset[str]) -> dict[str, object] | None: + """Load a small JSON object envelope, refusing anything of another shape.""" + try: + if path.stat().st_size > _MAX_ENVELOPE_PROBE_BYTES: + return None + with path.open("rb") as handle: + head = handle.read(1) + if head != b"{": + return None + handle.seek(0) + value = json.load(handle) + except (OSError, json.JSONDecodeError, RecursionError, ValueError): + return None + if not isinstance(value, dict) or not expected_keys.issubset(value): + return None + return value + + +class HookEventSpoolProver: + """Prove a hook-event envelope against the declared hook spool topology. + + Acquisition stores the *validated* record, whose ``observed_at_ms`` the + spool file does not carry, and both sides are serialized independently. + Byte equality is therefore the wrong law here: the proof is equality of + the production-route record, which is what admission would reproduce. + """ + + name = "hook-event-spool" + _ENVELOPE_KEYS = frozenset({"event_id", "event_type", "session_id", "timestamp", "provider", "payload"}) + + def __init__(self, sources: Sequence[tuple[str, Path]]) -> None: + self._sources = tuple(sources) + self._index: dict[str, tuple[str, Path]] | None = None + + def _spool_index(self) -> dict[str, tuple[str, Path]]: + if self._index is not None: + return self._index + index: dict[str, tuple[str, Path]] = {} + for source_id, root in self._sources: + for directory, subdirectories, filenames in os.walk(root): + subdirectories.sort() + for filename in sorted(filenames): + if not filename.endswith(".json"): + continue + index.setdefault(filename[: -len(".json")], (source_id, Path(directory) / filename)) + self._index = index + return index + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + envelope = _read_envelope(path, expected_keys=self._ENVELOPE_KEYS) + if envelope is None: + return None + event_id = envelope.get("event_id") + if not isinstance(event_id, str) or not event_id: + return None + located = self._spool_index().get(event_id) + if located is None: + return None + source_id, spool_path = located + from polylogue.sources.hooks import HookSpoolRecordError, read_hook_spool_record + + try: + record = read_hook_spool_record(spool_path) + except HookSpoolRecordError: + return None + if record != envelope: + return None + return SourceProof( + prover=self.name, + mode=SourceProofMode.SEMANTIC_EQUIVALENT, + source_id=source_id, + source_path=str(spool_path), + detail=f"hook event {event_id} reproduces through the spool read route", + ) + + def restoration_target(self, path: Path) -> RestorationTarget | None: + envelope = _read_envelope(path, expected_keys=self._ENVELOPE_KEYS) + if envelope is None: + return None + event_id = envelope.get("event_id") + if not isinstance(event_id, str) or not event_id: + return None + return RestorationTarget(destination=RestorationDestination.HOOK_EVENT_SPOOL, logical_id=event_id) + + +class BrowserCaptureSpoolProver: + """Prove a browser-capture envelope against the ordinary capture spool.""" + + name = "browser-capture-spool" + _ENVELOPE_KEYS = frozenset({"polylogue_capture_kind", "schema_version", "session", "provenance"}) + + def __init__(self, spool_root: Path, *, source_id: str = "browser-capture-spool") -> None: + self._spool_root = spool_root + self._source_id = source_id + + def _envelope(self, path: Path) -> object | None: + payload = _read_envelope(path, expected_keys=self._ENVELOPE_KEYS) + if payload is None: + return None + from pydantic import ValidationError + + from polylogue.browser_capture.models import BrowserCaptureEnvelope + + try: + return BrowserCaptureEnvelope.model_validate(payload) + except ValidationError: + return None + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + envelope = self._envelope(path) + if envelope is None: + return None + from polylogue.browser_capture.models import BrowserCaptureEnvelope + from polylogue.browser_capture.receiver import capture_artifact_path, capture_dedup_content_hash + + assert isinstance(envelope, BrowserCaptureEnvelope) + spooled = capture_artifact_path(envelope, self._spool_root) + if not spooled.is_file(): + return None + try: + existing = BrowserCaptureEnvelope.model_validate_json(spooled.read_bytes()) + except (OSError, ValueError): + return None + if capture_dedup_content_hash(existing) != capture_dedup_content_hash(envelope): + return None + return SourceProof( + prover=self.name, + mode=SourceProofMode.SEMANTIC_EQUIVALENT, + source_id=self._source_id, + source_path=str(spooled), + detail="capture spool holds a dedup-equivalent envelope", + ) + + def restoration_target(self, path: Path) -> RestorationTarget | None: + envelope = self._envelope(path) + if envelope is None: + return None + from polylogue.browser_capture.models import BrowserCaptureEnvelope + + assert isinstance(envelope, BrowserCaptureEnvelope) + return RestorationTarget( + destination=RestorationDestination.BROWSER_CAPTURE_SPOOL, + logical_id=f"{envelope.session.provider}:{envelope.session.provider_session_id}", + ) + + +def _hash_stream(handle: IO[bytes], *, limit: int | None = None) -> tuple[str, int]: + digest = hashlib.sha256() + consumed = 0 + while True: + want = _HASH_CHUNK_BYTES if limit is None else min(_HASH_CHUNK_BYTES, limit - consumed) + if want <= 0: + break + chunk = handle.read(want) + if not chunk: + break + digest.update(chunk) + consumed += len(chunk) + return digest.hexdigest(), consumed + + +@dataclass(frozen=True, slots=True) +class RawSourceCarrier: + """One acquisition's record of where a payload came from.""" + + source_path: str + append_start_offset: int | None = None + + +class RawSourceFileProver: + """Prove a raw payload against the source file it was acquired from. + + Three shapes all reproduce the content and all require a fresh hash: + the whole file, the file's own prefix (append-structured providers grow + in place), and the recorded append span for a row that captured only its + own increment. Path existence proves nothing. + """ + + name = "raw-source-file" + + def __init__(self, carriers_by_hash: Mapping[str, tuple[RawSourceCarrier, ...]]) -> None: + self._carriers = dict(carriers_by_hash) + + def _attempt(self, source: Path, *, offset: int, size_bytes: int, whole: bool) -> tuple[str, int] | None: + try: + with source.open("rb") as handle: + if offset: + handle.seek(offset) + return _hash_stream(handle, limit=None if whole else size_bytes) + except OSError: + return None + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + for carrier in self._carriers.get(blob_hash, ()): + source = Path(carrier.source_path) + try: + if not source.is_file(): + continue + source_size = source.stat().st_size + except OSError: + continue + attempts: list[tuple[SourceProofMode, int, bool]] = [] + if source_size == size_bytes: + attempts.append((SourceProofMode.BYTE_IDENTICAL, 0, True)) + elif source_size > size_bytes: + attempts.append((SourceProofMode.STRICT_PREFIX, 0, False)) + offset = carrier.append_start_offset + if offset is not None and offset > 0 and source_size >= offset + size_bytes: + attempts.append((SourceProofMode.STRICT_PREFIX, offset, False)) + for mode, start, whole in attempts: + measured = self._attempt(source, offset=start, size_bytes=size_bytes, whole=whole) + if measured is None: + continue + digest, consumed = measured + if consumed != size_bytes or digest != blob_hash: + continue + span = "whole file" if whole else f"{size_bytes} bytes at offset {start}" + return SourceProof( + prover=self.name, + mode=mode, + source_id="configured-source-file", + source_path=str(source), + detail=f"fresh hash over the {span} of the live source", + ) + return None + + +class AppendPrefixProver: + """Prove a blob is the exact prefix of a retained carrier of the same item. + + Scoped to carriers that share a logical source identity: an unrelated + object that merely happens to start with the same bytes is not append + lineage, and treating it as such would discard a distinct carrier. + """ + + name = "append-prefix" + + def __init__(self, successors_by_hash: Mapping[str, tuple[str, ...]], *, blob_store: BlobStore) -> None: + self._successors = dict(successors_by_hash) + self._store = blob_store + + def prove(self, blob_hash: str, path: Path, size_bytes: int) -> SourceProof | None: + for successor in self._successors.get(blob_hash, ()): + successor_path = self._store.blob_path(successor) + try: + if not successor_path.is_file() or successor_path.stat().st_size <= size_bytes: + continue + with successor_path.open("rb") as handle: + digest, consumed = _hash_stream(handle, limit=size_bytes) + except OSError: + continue + if consumed != size_bytes or digest != blob_hash: + continue + return SourceProof( + prover=self.name, + mode=SourceProofMode.STRICT_PREFIX, + source_id="retained-blob", + source_path=successor, + detail="exact prefix of a larger retained carrier of the same logical item", + ) + return None + + +@dataclass(frozen=True, slots=True) +class BlobDispositionContext: + """Everything a compilation needs, resolved once and reused per member.""" + + blob_store: BlobStore + provers: tuple[BlobSourceProver, ...] + referenced_hashes: frozenset[str] + restoration_provers: tuple[BlobRestorationResolver, ...] = field(default=()) + + +def _open_ro(path: Path) -> sqlite3.Connection: + return sqlite3.connect(f"file:{path}?mode=ro", uri=True) + + +def referenced_blob_hashes(source_db: Path) -> frozenset[str]: + """Union every durable relation that names a physical blob hash. + + A relation that exists but cannot be read is a failure, never an empty + set: reading zero references from an unreadable tier would license + deleting the whole namespace. + """ + relations = ( + ("blob_refs", "blob_hash"), + ("raw_sessions", "blob_hash"), + ("raw_hook_events", "blob_hash"), + ("raw_artifacts", "blob_hash"), + ("blob_publication_reservations", "blob_hash"), + ) + hashes: set[str] = set() + with closing(_open_ro(source_db)) as conn: + present = { + str(row[0]) + for row in conn.execute("SELECT name FROM sqlite_master WHERE type IN ('table','view')").fetchall() + } + for table, column in relations: + if table not in present: + continue + try: + columns = {str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})").fetchall()} + except sqlite3.Error as exc: + raise BlobDispositionError(f"reference relation {table} is unreadable: {exc}") from exc + if column not in columns: + continue + try: + rows = conn.execute( + f"SELECT DISTINCT lower(hex({column})) FROM {table} WHERE {column} IS NOT NULL" + ).fetchall() + except sqlite3.Error as exc: + raise BlobDispositionError(f"reference relation {table} is unreadable: {exc}") from exc + hashes.update(str(row[0]) for row in rows) + return frozenset(hashes) + + +def raw_source_carriers_by_hash(source_db: Path) -> dict[str, tuple[RawSourceCarrier, ...]]: + """Map each acquired payload hash to the source carriers that produced it.""" + mapping: dict[str, set[RawSourceCarrier]] = {} + with closing(_open_ro(source_db)) as conn: + try: + rows = conn.execute( + "SELECT lower(hex(blob_hash)), source_path, append_start_offset FROM raw_sessions " + "WHERE blob_hash IS NOT NULL AND source_path IS NOT NULL" + ).fetchall() + except sqlite3.Error as exc: + raise BlobDispositionError(f"raw_sessions is unreadable: {exc}") from exc + for blob_hash, source_path, offset in rows: + carrier = RawSourceCarrier(str(source_path), int(offset) if offset is not None else None) + mapping.setdefault(str(blob_hash), set()).add(carrier) + return { + key: tuple(sorted(value, key=lambda item: (item.source_path, item.append_start_offset or 0))) + for key, value in mapping.items() + } + + +def append_successors_by_hash(source_db: Path) -> dict[str, tuple[str, ...]]: + """Map each payload hash to larger carriers of the same logical item.""" + with closing(_open_ro(source_db)) as conn: + try: + rows = conn.execute( + "SELECT origin, native_id, lower(hex(blob_hash)), blob_size FROM raw_sessions " + "WHERE blob_hash IS NOT NULL AND native_id IS NOT NULL" + ).fetchall() + except sqlite3.Error as exc: + raise BlobDispositionError(f"raw_sessions is unreadable: {exc}") from exc + grouped: dict[tuple[str, str], list[tuple[int, str]]] = {} + for origin, native_id, blob_hash, size in rows: + if size is None: + continue + grouped.setdefault((str(origin), str(native_id)), []).append((int(size), str(blob_hash))) + successors: dict[str, tuple[str, ...]] = {} + for carriers in grouped.values(): + carriers.sort() + for index, (size, blob_hash) in enumerate(carriers): + larger = tuple(other for other_size, other in carriers[index + 1 :] if other_size > size) + if larger: + successors[blob_hash] = larger + return successors + + +def _restoration_target(path: Path, provers: Sequence[BlobRestorationResolver]) -> RestorationTarget | None: + for prover in provers: + target = prover.restoration_target(path) + if target is not None: + return target + return None + + +def classify_blob( + entry: BlobNamespaceEntry, + *, + context: BlobDispositionContext, +) -> BlobDispositionMember: + """Assign exactly one disposition to one physical blob.""" + assert entry.hash_hex is not None + blob_hash = entry.hash_hex + try: + size_bytes = entry.path.stat().st_size + except OSError as exc: + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=0, + referenced=blob_hash in context.referenced_hashes, + disposition=BlobDisposition.UNRESOLVED, + reason=f"physical object is unreadable: {exc}", + ) + referenced = blob_hash in context.referenced_hashes + for prover in context.provers: + proof = prover.prove(blob_hash, entry.path, size_bytes) + if proof is None: + continue + disposition = ( + BlobDisposition.SUPERSEDED_PREFIX + if proof.prover == AppendPrefixProver.name + else BlobDisposition.SOURCE_PRESENT + ) + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=size_bytes, + referenced=referenced, + disposition=disposition, + reason=f"{proof.prover} proved {proof.mode.value}", + proof=proof, + ) + restoration = _restoration_target(entry.path, context.restoration_provers) + if restoration is not None: + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=size_bytes, + referenced=referenced, + disposition=BlobDisposition.RESTORE_REQUIRED, + reason="no configured source holds this content and it names an ordinary spool destination", + restoration=restoration, + ) + return BlobDispositionMember( + blob_hash=blob_hash, + size_bytes=size_bytes, + referenced=referenced, + disposition=BlobDisposition.UNRESOLVED, + reason="no source proof and no ordinary restoration destination", + ) + + +def resolve_disposition_roots(archive_root: Path) -> tuple[Path, tuple[tuple[str, Path], ...], Path]: + """Resolve the primary hook spool, the declared spool topology, and captures. + + The declared topology already includes the legacy read-only roots, so a + carrier whose event still sits in a superseded spool is proven at a + configured source rather than restored a second time. + """ + from polylogue.sources.hooks import hook_spool_sources + + hooks_root = archive_root / "hooks" + sources = tuple( + (spec.source_id, spec.root) for spec in hook_spool_sources(primary_root=hooks_root) if spec.root.is_dir() + ) + return hooks_root, sources, archive_root / "browser-capture" + + +def build_disposition_context( + *, + archive_root: Path, + blob_root: Path, + source_db: Path, + hook_spool_sources: Sequence[tuple[str, Path]], + browser_capture_spool: Path, +) -> BlobDispositionContext: + """Resolve the prover set from configured sources, not from history.""" + store = BlobStore(blob_root) + hook_prover = HookEventSpoolProver(hook_spool_sources) + capture_prover = BrowserCaptureSpoolProver(browser_capture_spool) + provers: tuple[BlobSourceProver, ...] = ( + hook_prover, + capture_prover, + RawSourceFileProver(raw_source_carriers_by_hash(source_db)), + AppendPrefixProver(append_successors_by_hash(source_db), blob_store=store), + ) + return BlobDispositionContext( + blob_store=store, + provers=provers, + referenced_hashes=referenced_blob_hashes(source_db), + restoration_provers=(hook_prover, capture_prover), + ) + + +def compile_disposition_plan( + *, + archive_root: Path, + blob_root: Path, + source_db: Path, + context: BlobDispositionContext | None = None, + hook_spool_sources: Sequence[tuple[str, Path]] | None = None, + browser_capture_spool: Path | None = None, + progress: object | None = None, +) -> BlobDispositionPlan: + """Walk the complete physical namespace and compile one immutable plan.""" + if context is None: + if hook_spool_sources is None or browser_capture_spool is None: + raise BlobDispositionError("compilation needs either a context or the configured spool roots") + context = build_disposition_context( + archive_root=archive_root, + blob_root=blob_root, + source_db=source_db, + hook_spool_sources=hook_spool_sources, + browser_capture_spool=browser_capture_spool, + ) + members: list[BlobDispositionMember] = [] + invalid: list[str] = [] + seen: set[str] = set() + file_count = 0 + for entry in context.blob_store.iter_namespace(): + if entry.kind is not BlobNamespaceEntryKind.BLOB: + invalid.append(f"{entry.relative_path}: {entry.issue.value if entry.issue else 'unclassified'}") + continue + file_count += 1 + assert entry.hash_hex is not None + if entry.hash_hex in seen: + continue + seen.add(entry.hash_hex) + members.append(classify_blob(entry, context=context)) + if progress is not None and len(members) % 1000 == 0: + progress(len(members)) # type: ignore[operator] + present = frozenset(seen) + denominator = BlobDispositionDenominator( + physical_file_count=file_count, + distinct_hash_count=len(members), + total_bytes=sum(member.size_bytes for member in members), + referenced_hash_count=len(context.referenced_hashes), + referenced_present_count=len(context.referenced_hashes & present), + referenced_absent_count=len(context.referenced_hashes - present), + invalid_namespace_entries=tuple(sorted(invalid)), + ) + return BlobDispositionPlan( + tool_version=TOOL_VERSION, + archive_root=str(archive_root), + blob_root=str(blob_root), + denominator=denominator, + members=tuple(sorted(members, key=lambda member: member.blob_hash)), + ) + + +__all__ = [ + "TOOL_VERSION", + "AppendPrefixProver", + "BlobDisposition", + "BlobDispositionContext", + "BlobDispositionDenominator", + "BlobDispositionError", + "BlobDispositionMember", + "BlobDispositionPlan", + "BlobRestorationResolver", + "BlobSourceProver", + "BrowserCaptureSpoolProver", + "HookEventSpoolProver", + "RawSourceCarrier", + "RawSourceFileProver", + "RestorationDestination", + "RestorationTarget", + "SourceProof", + "SourceProofMode", + "append_successors_by_hash", + "build_disposition_context", + "classify_blob", + "compile_disposition_plan", + "raw_source_carriers_by_hash", + "resolve_disposition_roots", + "referenced_blob_hashes", +] diff --git a/polylogue/maintenance/blob_disposition_apply.py b/polylogue/maintenance/blob_disposition_apply.py new file mode 100644 index 0000000000..e0b70a06fe --- /dev/null +++ b/polylogue/maintenance/blob_disposition_apply.py @@ -0,0 +1,475 @@ +"""Consume one accepted blob disposition plan under explicit authorization. + +Two effects, in one order that cannot be reversed: + +1. **Restore** every ``restore_required`` member into its ordinary spool + through the production receiver that admission already reads. Restoration + never touches the physical blob: the historical carrier survives this + module unconditionally, so a crash at any boundary leaves at least one + verified copy. +2. **Delete** ``source_present`` and ``superseded_prefix`` members through + the canonical blob-GC seam, which owns publisher exclusion, the final + locked liveness recheck, and crash-consistent generation intent. + +The plan is a capability, not a worklist. This module makes no classification +judgment: every member's proof is revalidated immediately before its effect, +and any drift — a changed source, a changed object, a new referent, a +different digest, a different denominator — invalidates the whole plan and +returns control to compilation. + +Deletion is bounded to unreferenced members by construction. A member whose +content is proven at its source but which a durable row still references +stays on disk; removing it is the reference owner's decision, and the GC seam +refuses it anyway. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path + +from polylogue.maintenance.blob_disposition import ( + BlobDisposition, + BlobDispositionContext, + BlobDispositionMember, + BlobDispositionPlan, + RestorationDestination, +) + +TOOL_VERSION = "blob-disposition-apply-v1" + + +class DispositionApplyError(RuntimeError): + """Raised when an apply cannot prove its exact authorized effect set.""" + + +class MemberOutcome(StrEnum): + """One terminal outcome per plan member. There is no unknown outcome.""" + + RESTORED = "restored" + RESTORATION_ALREADY_PRESENT = "restoration_already_present" + DELETED = "deleted" + RETAINED_REFERENCED = "retained_referenced" + RETAINED_ABSENT = "retained_absent" + BLOCKED = "blocked" + + +@dataclass(frozen=True, slots=True) +class MemberResult: + blob_hash: str + outcome: MemberOutcome + detail: str = "" + + def to_dict(self) -> dict[str, str]: + return {"blob_hash": self.blob_hash, "outcome": self.outcome.value, "detail": self.detail} + + +@dataclass(frozen=True, slots=True) +class DispositionApplyReceipt: + """Complete before/after evidence, derived only from member outcomes.""" + + tool_version: str + plan_digest: str + archive_root: str + blob_root: str + dry_run: bool + results: tuple[MemberResult, ...] + reclaimed_bytes: int = 0 + blockers: tuple[str, ...] = () + + @property + def counts(self) -> dict[str, int]: + counts = {outcome.value: 0 for outcome in MemberOutcome} + for result in self.results: + counts[result.outcome.value] += 1 + return counts + + @property + def ok(self) -> bool: + return not self.blockers and self.counts[MemberOutcome.BLOCKED.value] == 0 + + def to_dict(self) -> dict[str, object]: + return { + "tool_version": self.tool_version, + "plan_digest": self.plan_digest, + "archive_root": self.archive_root, + "blob_root": self.blob_root, + "dry_run": self.dry_run, + "ok": self.ok, + "counts": self.counts, + "reclaimed_bytes": self.reclaimed_bytes, + "blockers": list(self.blockers), + "results": [result.to_dict() for result in self.results], + } + + +def _revalidate(member: BlobDispositionMember, *, context: BlobDispositionContext) -> str | None: + """Re-derive the member's own proof at the moment of effect.""" + path = context.blob_store.blob_path(member.blob_hash) + if not path.is_file(): + return "physical object vanished between planning and apply" + try: + size_bytes = path.stat().st_size + except OSError as exc: + return f"physical object became unreadable: {exc}" + if size_bytes != member.size_bytes: + return f"physical object changed size {member.size_bytes} -> {size_bytes}" + referenced_now = member.blob_hash in context.referenced_hashes + if referenced_now and not member.referenced: + return "a new durable reference appeared after planning" + if member.disposition is BlobDisposition.RESTORE_REQUIRED: + for prover in context.provers: + if prover.prove(member.blob_hash, path, size_bytes) is not None: + return "a source proof appeared after planning; restoration is no longer justified" + return None + expected = member.proof + if expected is None: + return "member carries no proof to revalidate" + for prover in context.provers: + if prover.name != expected.prover: + continue + proof = prover.prove(member.blob_hash, path, size_bytes) + if proof is None: + return f"{expected.prover} no longer proves this object at its source" + if proof.source_path != expected.source_path or proof.mode is not expected.mode: + return f"{expected.prover} now proves a different source or mode" + return None + return f"prover {expected.prover} is not available at apply time" + + +def _resident_hook_event(spool_root: Path, event_id: str) -> Path | None: + """Locate an event anywhere in the spool, not only in today's shard. + + ``enqueue_hook_event`` shards by the current day and only refuses a + collision inside that shard, so a same-identity event spooled on another + day would be delivered twice. + """ + for candidate in sorted(spool_root.rglob(f"{event_id}.json")): + if candidate.is_file(): + return candidate + return None + + +def _restore_hook_event(member: BlobDispositionMember, *, path: Path, spool_root: Path) -> MemberResult: + from polylogue.sources.hooks import ( + HookSpoolRecordError, + enqueue_hook_event, + read_hook_spool_record, + ) + + try: + envelope = json.loads(path.read_bytes()) + except (OSError, json.JSONDecodeError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"carrier is not a readable envelope: {exc}") + if not isinstance(envelope, dict): + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "carrier envelope is not an object") + event_id = envelope.get("event_id") + if not isinstance(event_id, str) or not event_id: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "carrier envelope has no event identity") + resident = _resident_hook_event(spool_root, event_id) + if resident is not None: + try: + existing = read_hook_spool_record(resident) + except HookSpoolRecordError as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"destination is unreadable: {exc}") + if existing != envelope: + return MemberResult( + member.blob_hash, + MemberOutcome.BLOCKED, + "destination holds a different event under the same identity", + ) + return MemberResult(member.blob_hash, MemberOutcome.RESTORATION_ALREADY_PRESENT, str(resident)) + try: + published = enqueue_hook_event( + event_type=str(envelope["event_type"]), + session_id=str(envelope["session_id"]), + provider=str(envelope["provider"]), + timestamp=str(envelope["timestamp"]), + payload=dict(envelope["payload"]), + root=spool_root, + event_id=str(envelope["event_id"]), + ) + except (KeyError, TypeError, HookSpoolRecordError, OSError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"ordinary spool admission refused: {exc}") + try: + restored = read_hook_spool_record(published) + except HookSpoolRecordError as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"restored file does not read back: {exc}") + if restored != envelope: + return MemberResult( + member.blob_hash, + MemberOutcome.BLOCKED, + "destination holds a different event under the same identity", + ) + _fsync_directory(published.parent) + return MemberResult(member.blob_hash, MemberOutcome.RESTORED, str(published)) + + +def _restore_browser_capture(member: BlobDispositionMember, *, path: Path, spool_root: Path) -> MemberResult: + from pydantic import ValidationError + + from polylogue.browser_capture.models import BrowserCaptureEnvelope + from polylogue.browser_capture.receiver import ( + BrowserCaptureSpoolConflictError, + SpoolQuotaExceededError, + capture_artifact_path, + capture_dedup_content_hash, + write_capture_envelope_bytes, + ) + + try: + raw = path.read_bytes() + envelope = BrowserCaptureEnvelope.model_validate_json(raw) + except (OSError, ValidationError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"carrier is not a valid capture: {exc}") + destination = capture_artifact_path(envelope, spool_root) + if destination.is_file(): + try: + existing = BrowserCaptureEnvelope.model_validate_json(destination.read_bytes()) + except (OSError, ValidationError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"destination is unreadable: {exc}") + if capture_dedup_content_hash(existing) != capture_dedup_content_hash(envelope): + return MemberResult( + member.blob_hash, + MemberOutcome.BLOCKED, + "destination holds a different capture under the same identity", + ) + return MemberResult(member.blob_hash, MemberOutcome.RESTORATION_ALREADY_PRESENT, str(destination)) + try: + write_capture_envelope_bytes(raw, spool_path=spool_root) + except (BrowserCaptureSpoolConflictError, SpoolQuotaExceededError, OSError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"ordinary spool admission refused: {exc}") + if not destination.is_file(): + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "capture receiver published no artifact") + try: + restored = BrowserCaptureEnvelope.model_validate_json(destination.read_bytes()) + except (OSError, ValidationError, ValueError) as exc: + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, f"restored file does not read back: {exc}") + if capture_dedup_content_hash(restored) != capture_dedup_content_hash(envelope): + return MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "restored capture is not content-equivalent") + _fsync_directory(destination.parent) + return MemberResult(member.blob_hash, MemberOutcome.RESTORED, str(destination)) + + +def _fsync_directory(path: Path) -> None: + """Persist the atomic rename's directory entry before claiming success.""" + try: + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + except OSError: + return + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def restore_plan_members( + plan: BlobDispositionPlan, + *, + context: BlobDispositionContext, + hook_spool_root: Path, + browser_capture_spool: Path, + dry_run: bool = True, +) -> tuple[MemberResult, ...]: + """Restore every sole-copy carrier into its ordinary spool. + + This never deletes or modifies the historical carrier, so an interruption + at any point leaves the blob intact and the operation resumable. + """ + results: list[MemberResult] = [] + for member in plan.members_for(BlobDisposition.RESTORE_REQUIRED): + drift = _revalidate(member, context=context) + if drift is not None: + results.append(MemberResult(member.blob_hash, MemberOutcome.BLOCKED, drift)) + continue + if member.restoration is None: + results.append( + MemberResult(member.blob_hash, MemberOutcome.BLOCKED, "restore_required member names no destination") + ) + continue + if dry_run: + results.append( + MemberResult( + member.blob_hash, + MemberOutcome.RESTORED, + f"would restore to {member.restoration.destination.value}", + ) + ) + continue + path = context.blob_store.blob_path(member.blob_hash) + if member.restoration.destination is RestorationDestination.HOOK_EVENT_SPOOL: + results.append(_restore_hook_event(member, path=path, spool_root=hook_spool_root)) + else: + results.append(_restore_browser_capture(member, path=path, spool_root=browser_capture_spool)) + return tuple(results) + + +def _authorization_blockers( + plan: BlobDispositionPlan, + *, + authorized_digest: str, + context: BlobDispositionContext, +) -> tuple[str, ...]: + blockers: list[str] = [] + if not plan.accepted: + blockers.append(f"plan is not acceptable: {plan.unresolved_count} unresolved members") + actual = plan.digest() + if actual != authorized_digest: + blockers.append(f"authorized digest {authorized_digest[:16]} does not match plan digest {actual[:16]}") + if str(context.blob_store.root) != plan.blob_root: + blockers.append(f"plan blob namespace {plan.blob_root} is not the namespace being applied") + referenced_present = len(context.referenced_hashes & {member.blob_hash for member in plan.members}) + if referenced_present != plan.denominator.referenced_present_count: + blockers.append( + "referenced-and-present denominator drifted " + f"{plan.denominator.referenced_present_count} -> {referenced_present}" + ) + return tuple(blockers) + + +def apply_disposition_plan( + plan: BlobDispositionPlan, + *, + context: BlobDispositionContext, + authorized_digest: str, + source_db: Path, + index_db: Path, + hook_spool_root: Path, + browser_capture_spool: Path, + writer_block_reason: str | None = None, + dry_run: bool = True, +) -> DispositionApplyReceipt: + """Restore, then delete, exactly what the authorized plan names.""" + blockers = list(_authorization_blockers(plan, authorized_digest=authorized_digest, context=context)) + if writer_block_reason is not None and not dry_run: + blockers.append(f"an archive writer is active: {writer_block_reason}") + if blockers: + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=(), + blockers=tuple(blockers), + ) + + results: list[MemberResult] = list( + restore_plan_members( + plan, + context=context, + hook_spool_root=hook_spool_root, + browser_capture_spool=browser_capture_spool, + dry_run=dry_run, + ) + ) + if any(result.outcome is MemberOutcome.BLOCKED for result in results): + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=tuple(results), + blockers=("restoration did not complete; no deletion was attempted",), + ) + + removable: list[BlobDispositionMember] = [] + for disposition in (BlobDisposition.SOURCE_PRESENT, BlobDisposition.SUPERSEDED_PREFIX): + for member in plan.members_for(disposition): + drift = _revalidate(member, context=context) + if drift is not None: + results.append(MemberResult(member.blob_hash, MemberOutcome.BLOCKED, drift)) + continue + if member.referenced or member.blob_hash in context.referenced_hashes: + results.append( + MemberResult( + member.blob_hash, + MemberOutcome.RETAINED_REFERENCED, + "content is proven at its source but a durable row still references the object", + ) + ) + continue + removable.append(member) + + if any(result.outcome is MemberOutcome.BLOCKED for result in results): + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=tuple(results), + blockers=("member revalidation failed; no deletion was attempted",), + ) + + # An empty removable set has no effect to serialize: entering the GC seam + # would only report its own unmet preconditions as this plan's blockers. + if dry_run or not removable: + results.extend( + MemberResult(member.blob_hash, MemberOutcome.DELETED, "would unlink through the blob-GC seam") + for member in removable + ) + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=dry_run, + results=tuple(results), + reclaimed_bytes=sum(member.size_bytes for member in removable) if dry_run else 0, + ) + + from polylogue.storage.blob_gc import unlink_unreferenced_blob_hashes_under_exclusion + + deleted, reclaimed, errors = unlink_unreferenced_blob_hashes_under_exclusion( + source_db, + index_db, + context.blob_store.root, + {member.blob_hash for member in removable}, + ) + for member in removable: + if context.blob_store.blob_path(member.blob_hash).exists(): + results.append( + MemberResult(member.blob_hash, MemberOutcome.RETAINED_ABSENT, "the GC seam declined this member") + ) + else: + results.append(MemberResult(member.blob_hash, MemberOutcome.DELETED, "")) + return DispositionApplyReceipt( + tool_version=TOOL_VERSION, + plan_digest=plan.digest(), + archive_root=plan.archive_root, + blob_root=plan.blob_root, + dry_run=False, + results=tuple(results), + reclaimed_bytes=reclaimed, + blockers=tuple(errors), + ) + + +def write_receipt(path: Path, receipt: DispositionApplyReceipt) -> None: + """Publish an append-only receipt, durably, before returning success.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.partial") + payload = json.dumps(receipt.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + "\n" + with temporary.open("w", encoding="utf-8") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + _fsync_directory(path.parent) + + +__all__ = [ + "TOOL_VERSION", + "DispositionApplyError", + "DispositionApplyReceipt", + "MemberOutcome", + "MemberResult", + "apply_disposition_plan", + "restore_plan_members", + "write_receipt", +] diff --git a/polylogue/sources/hooks.py b/polylogue/sources/hooks.py index e11b1c5ef9..b17eaa4ce9 100644 --- a/polylogue/sources/hooks.py +++ b/polylogue/sources/hooks.py @@ -446,6 +446,18 @@ def _prune_empty_shards(shards: set[Path], pending_root: Path) -> None: shard.rmdir() +def read_hook_spool_record(path: Path) -> dict[str, object]: + """Read one spool file through the same validation acquisition applies. + + The record acquisition stores is not the file's bytes: ``observed_at_ms`` + is derived here, and serialization is independent on both sides. Any + comparison against stored hook material must go through this route rather + than compare bytes. + """ + + return _read_record(path) + + def _read_record(path: Path) -> dict[str, object]: try: value = json.loads(path.read_text(encoding="utf-8")) @@ -615,5 +627,6 @@ def _fsync_directory(path: Path) -> None: "enqueue_hook_event", "hook_spool_root", "pending_hook_spool_dir", + "read_hook_spool_record", "validate_hook_spool_topology", ] diff --git a/tests/unit/maintenance/test_blob_disposition_apply.py b/tests/unit/maintenance/test_blob_disposition_apply.py new file mode 100644 index 0000000000..9f85ad777b --- /dev/null +++ b/tests/unit/maintenance/test_blob_disposition_apply.py @@ -0,0 +1,426 @@ +"""Fault matrix for consuming an accepted blob disposition plan. + +The apply boundary is irreversible, so every test here names the mutation +that would make it red: deleting before restoring, trusting a stale plan, +accepting a changed source or denominator, or converting a blocked member +into a silent success. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +from polylogue.maintenance.blob_disposition import ( + BlobDisposition, + BlobDispositionContext, + BlobDispositionPlan, + build_disposition_context, + compile_disposition_plan, +) +from polylogue.maintenance.blob_disposition_apply import ( + MemberOutcome, + apply_disposition_plan, + restore_plan_members, + write_receipt, +) +from polylogue.sources.hooks import read_hook_spool_record +from polylogue.storage.blob_store import BlobStore + + +def _hook_envelope(event_id: str = "event-1", *, text: str = "ran a tool") -> dict[str, object]: + return { + "event_id": event_id, + "event_type": "PreToolUse", + "session_id": "session-1", + "timestamp": "2026-07-15T02:15:39Z", + "provider": "claude-code", + "payload": {"tool_name": "Bash", "detail": text}, + } + + +def _stored_bytes(envelope: dict[str, object], tmp_path: Path) -> bytes: + scratch = tmp_path / f"scratch-{envelope['event_id']}.json" + scratch.write_text(json.dumps(envelope, sort_keys=True), encoding="utf-8") + return json.dumps(read_hook_spool_record(scratch), ensure_ascii=False, sort_keys=True, indent=1).encode("utf-8") + + +def _write_spool_file(root: Path, envelope: dict[str, object]) -> Path: + target = root / "pending" / "2026-07-15" + target.mkdir(parents=True, exist_ok=True) + path = target / f"{envelope['event_id']}.json" + path.write_text(json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=4), encoding="utf-8") + return path + + +def _archive(tmp_path: Path) -> tuple[Path, Path, Path, Path]: + archive_root = tmp_path / "archive" + blob_root = archive_root / "blob" + blob_root.mkdir(parents=True) + hooks_root = archive_root / "hooks" + hooks_root.mkdir() + capture_spool = archive_root / "browser-capture" + capture_spool.mkdir() + source_db = archive_root / "source.db" + with sqlite3.connect(source_db) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB, ref_type TEXT)") + conn.execute( + "CREATE TABLE raw_sessions (raw_id TEXT, origin TEXT, native_id TEXT, blob_hash BLOB, " + "blob_size INTEGER, source_path TEXT, append_start_offset INTEGER)" + ) + with sqlite3.connect(archive_root / "index.db") as conn: + conn.execute("CREATE TABLE sessions (session_id TEXT)") + return archive_root, blob_root, hooks_root, capture_spool + + +def _plan_and_context( + archive_root: Path, + blob_root: Path, + *, + legacy_root: Path | None = None, + capture_spool: Path, +) -> tuple[BlobDispositionPlan, BlobDispositionContext]: + hook_sources = (("legacy-hook-spool-0", legacy_root),) if legacy_root is not None else () + context = build_disposition_context( + archive_root=archive_root, + blob_root=blob_root, + source_db=archive_root / "source.db", + hook_spool_sources=hook_sources, + browser_capture_spool=capture_spool, + ) + plan = compile_disposition_plan( + archive_root=archive_root, + blob_root=blob_root, + source_db=archive_root / "source.db", + context=context, + ) + return plan, context + + +def test_restoration_publishes_into_the_ordinary_spool_and_keeps_the_carrier(tmp_path: Path) -> None: + """Anti-vacuity: deleting the carrier during restoration makes this red.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + envelope = _hook_envelope("sole-copy") + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + assert plan.members[0].disposition is BlobDisposition.RESTORE_REQUIRED + + (result,) = restore_plan_members( + plan, + context=context, + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert result.outcome is MemberOutcome.RESTORED + restored = Path(result.detail) + assert restored.is_file() + assert read_hook_spool_record(restored) == json.loads(store.blob_path(blob_hash).read_bytes()) + assert store.blob_path(blob_hash).is_file() + + +def test_restoration_is_idempotent_by_logical_identity(tmp_path: Path) -> None: + """Anti-vacuity: matching only today's day shard double-delivers a retry. + + ``enqueue_hook_event`` refuses a collision inside the current day's shard + only, so a resident event spooled on any other day must be found by + identity or the retry writes a second carrier of the same event. + """ + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + envelope = _hook_envelope("sole-copy") + store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + + (first,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + # Relocate the restored carrier into another day's shard: the retry must + # still recognize it rather than publish a second copy. + relocated = hooks_root / "pending" / "2026-07-15" + relocated.mkdir(parents=True, exist_ok=True) + Path(first.detail).rename(relocated / "sole-copy.json") + + (second,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert first.outcome is MemberOutcome.RESTORED + assert second.outcome is MemberOutcome.RESTORATION_ALREADY_PRESENT + assert [path.name for path in hooks_root.rglob("*.json")] == ["sole-copy.json"] + + +def test_restoration_blocks_on_a_hostile_collision(tmp_path: Path) -> None: + """Anti-vacuity: overwriting on identity collision loses the resident event.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + store.write_from_bytes(_stored_bytes(_hook_envelope("collide", text="the stored call"), tmp_path)) + resident = hooks_root / "pending" / "2026-07-15" + resident.mkdir(parents=True) + (resident / "collide.json").write_text( + json.dumps(_hook_envelope("collide", text="a different call"), sort_keys=True), encoding="utf-8" + ) + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + + (result,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert result.outcome is MemberOutcome.BLOCKED + assert "different event" in result.detail + assert json.loads((resident / "collide.json").read_text())["payload"]["detail"] == "a different call" + + +def test_a_source_proof_appearing_after_planning_blocks_restoration(tmp_path: Path) -> None: + """Anti-vacuity: skipping revalidation restores material already at its source.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + legacy_root.mkdir() + store = BlobStore(blob_root) + envelope = _hook_envelope("late-arrival") + store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, _ = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + assert plan.members[0].disposition is BlobDisposition.RESTORE_REQUIRED + + _write_spool_file(legacy_root, envelope) + _, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + (result,) = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert result.outcome is MemberOutcome.BLOCKED + assert "no longer justified" in result.detail + + +def test_a_stale_authorized_digest_refuses_before_any_effect(tmp_path: Path) -> None: + """Anti-vacuity: applying without digest binding consumes an edited plan.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest="0" * 64, + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert not receipt.ok + assert any("does not match plan digest" in blocker for blocker in receipt.blockers) + assert store.blob_path(blob_hash).is_file() + + +def test_an_unresolved_member_refuses_the_whole_plan(tmp_path: Path) -> None: + """Anti-vacuity: applying a partially explained plan deletes beside a mystery.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + proven_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + mystery_hash, _ = store.write_from_bytes(b"%PDF-1.5\nunexplained\n") + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + assert plan.unresolved_count == 1 + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert not receipt.ok + assert any("not acceptable" in blocker for blocker in receipt.blockers) + assert store.blob_path(proven_hash).is_file() + assert store.blob_path(mystery_hash).is_file() + + +def test_an_active_writer_refuses_an_active_apply(tmp_path: Path) -> None: + """Anti-vacuity: unserialized apply races the archive's single writer.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + writer_block_reason="live pidfile PID 4242 is running", + dry_run=False, + ) + + assert not receipt.ok + assert any("writer is active" in blocker for blocker in receipt.blockers) + assert store.blob_path(blob_hash).is_file() + + +def test_a_changed_source_invalidates_the_member_before_deletion(tmp_path: Path) -> None: + """Anti-vacuity: trusting the planning-time proof deletes divergent material.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + spool_file = _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + plan, _ = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + assert plan.accepted + + spool_file.write_text( + json.dumps(_hook_envelope("proven", text="rewritten at the source"), sort_keys=True), encoding="utf-8" + ) + _, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert not receipt.ok + assert store.blob_path(blob_hash).is_file() + assert any(result.outcome is MemberOutcome.BLOCKED for result in receipt.results) + + +def test_a_referenced_object_is_retained_not_deleted(tmp_path: Path) -> None: + """Anti-vacuity: deleting a proven-but-referenced object breaks a live row.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope("proven") + _write_spool_file(legacy_root, envelope) + store = BlobStore(blob_root) + blob_hash, _ = store.write_from_bytes(_stored_bytes(envelope, tmp_path)) + with sqlite3.connect(archive_root / "source.db") as conn: + conn.execute("INSERT INTO blob_refs (blob_hash, ref_type) VALUES (?, ?)", (bytes.fromhex(blob_hash), "raw")) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + + assert receipt.ok + assert [result.outcome for result in receipt.results] == [MemberOutcome.RETAINED_REFERENCED] + assert store.blob_path(blob_hash).is_file() + + +def test_a_dry_rehearsal_touches_nothing(tmp_path: Path) -> None: + """Anti-vacuity: a rehearsal that wrote would make the review meaningless.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + proven = _hook_envelope("proven") + _write_spool_file(legacy_root, proven) + store = BlobStore(blob_root) + proven_hash, _ = store.write_from_bytes(_stored_bytes(proven, tmp_path)) + sole_hash, _ = store.write_from_bytes(_stored_bytes(_hook_envelope("sole-copy"), tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=True, + ) + + assert receipt.ok and receipt.dry_run + assert store.blob_path(proven_hash).is_file() + assert store.blob_path(sole_hash).is_file() + assert list(hooks_root.rglob("*.json")) == [] + + +def test_receipt_totals_derive_from_member_outcomes(tmp_path: Path) -> None: + """Anti-vacuity: a summary counter maintained beside the members can drift.""" + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + legacy_root = tmp_path / "legacy-hooks" + _write_spool_file(legacy_root, _hook_envelope("proven")) + store = BlobStore(blob_root) + store.write_from_bytes(_stored_bytes(_hook_envelope("proven"), tmp_path)) + store.write_from_bytes(_stored_bytes(_hook_envelope("sole-copy"), tmp_path)) + plan, context = _plan_and_context(archive_root, blob_root, legacy_root=legacy_root, capture_spool=capture_spool) + + receipt = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=True, + ) + + assert sum(receipt.counts.values()) == len(receipt.results) == len(plan.members) + destination = tmp_path / "receipts" / "disposition.json" + write_receipt(destination, receipt) + assert json.loads(destination.read_text())["counts"] == receipt.counts + + +def test_restoration_proceeds_while_other_members_are_unresolved(tmp_path: Path) -> None: + """Anti-vacuity: gating restoration on plan acceptance strands sole copies. + + Restoration never removes a carrier, so an unrelated unexplained object + must not delay preserving the only copy of wanted material. + """ + archive_root, blob_root, hooks_root, capture_spool = _archive(tmp_path) + store = BlobStore(blob_root) + sole_hash, _ = store.write_from_bytes(_stored_bytes(_hook_envelope("sole-copy"), tmp_path)) + store.write_from_bytes(b"%PDF-1.5\nunexplained\n") + plan, context = _plan_and_context(archive_root, blob_root, capture_spool=capture_spool) + assert not plan.accepted + + results = restore_plan_members( + plan, context=context, hook_spool_root=hooks_root, browser_capture_spool=capture_spool, dry_run=False + ) + + assert [result.outcome for result in results] == [MemberOutcome.RESTORED] + assert store.blob_path(sole_hash).is_file() + + refused = apply_disposition_plan( + plan, + context=context, + authorized_digest=plan.digest(), + source_db=archive_root / "source.db", + index_db=archive_root / "index.db", + hook_spool_root=hooks_root, + browser_capture_spool=capture_spool, + dry_run=False, + ) + assert not refused.ok diff --git a/tests/unit/maintenance/test_blob_disposition_plan.py b/tests/unit/maintenance/test_blob_disposition_plan.py new file mode 100644 index 0000000000..71cf3df995 --- /dev/null +++ b/tests/unit/maintenance/test_blob_disposition_plan.py @@ -0,0 +1,338 @@ +"""Laws for the physical blob disposition plan. + +Every test names the mutation that makes it red. The plan decides whether an +irreplaceable object is deleted, so the anti-vacuity conditions are all of the +same family: a prover that accepts material current sources do not hold, or a +classifier that converts "unknown" into "discard", must fail here. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest + +from polylogue.maintenance.blob_disposition import ( + AppendPrefixProver, + BlobDisposition, + BlobDispositionContext, + BlobDispositionError, + BlobDispositionPlan, + RawSourceCarrier, + RawSourceFileProver, + RestorationDestination, + SourceProofMode, + append_successors_by_hash, + build_disposition_context, + compile_disposition_plan, + raw_source_carriers_by_hash, + referenced_blob_hashes, +) +from polylogue.storage.blob_store import BlobStore + + +def _hook_envelope(event_id: str = "event-1", *, text: str = "ran a tool") -> dict[str, object]: + return { + "event_id": event_id, + "event_type": "PreToolUse", + "session_id": "session-1", + "timestamp": "2026-07-15T02:15:39Z", + "provider": "claude-code", + "payload": {"tool_name": "Bash", "detail": text}, + } + + +def _write_spool_file(root: Path, envelope: dict[str, object], *, indent: int | None = None) -> Path: + target = root / "pending" / "2026-07-15" + target.mkdir(parents=True, exist_ok=True) + path = target / f"{envelope['event_id']}.json" + path.write_text(json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=indent), encoding="utf-8") + return path + + +def _publish_blob(store: BlobStore, payload: bytes) -> str: + blob_hash, _ = store.write_from_bytes(payload) + return blob_hash + + +def _stored_envelope_bytes(spool_file: Path) -> bytes: + """Serialize the validated record the way acquisition stored it.""" + from polylogue.sources.hooks import read_hook_spool_record + + record = read_hook_spool_record(spool_file) + return json.dumps(record, ensure_ascii=False, sort_keys=True, indent=1).encode("utf-8") + + +def _empty_source_db(path: Path) -> Path: + with sqlite3.connect(path) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB, ref_type TEXT)") + conn.execute( + "CREATE TABLE raw_sessions (raw_id TEXT, origin TEXT, native_id TEXT, blob_hash BLOB, " + "blob_size INTEGER, source_path TEXT, append_start_offset INTEGER)" + ) + return path + + +def _context(tmp_path: Path, *, hook_roots: tuple[tuple[str, Path], ...] = ()) -> BlobDispositionContext: + blob_root = tmp_path / "blob" + blob_root.mkdir(exist_ok=True) + source_db = _empty_source_db(tmp_path / "source.db") + return build_disposition_context( + archive_root=tmp_path, + blob_root=blob_root, + source_db=source_db, + hook_spool_sources=hook_roots, + browser_capture_spool=tmp_path / "browser-capture", + ) + + +def test_hook_envelope_is_source_present_despite_differing_bytes(tmp_path: Path) -> None: + """Anti-vacuity: a byte-equality prover would call this a sole copy and delete it. + + Acquisition derives ``observed_at_ms`` and both sides serialize + independently, so the stored object never equals the spool file's bytes. + """ + spool_root = tmp_path / "legacy-hooks" + envelope = _hook_envelope() + spool_file = _write_spool_file(spool_root, envelope, indent=4) + store = BlobStore(tmp_path / "blob") + blob_hash = _publish_blob(store, _stored_envelope_bytes(spool_file)) + assert store.blob_path(blob_hash).read_bytes() != spool_file.read_bytes() + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, + blob_root=store.root, + source_db=tmp_path / "source.db", + context=context, + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.SOURCE_PRESENT + assert member.proof is not None + assert member.proof.mode is SourceProofMode.SEMANTIC_EQUIVALENT + assert member.proof.source_path == str(spool_file) + assert plan.accepted + + +def test_hook_envelope_without_a_spool_file_is_restore_required(tmp_path: Path) -> None: + """Anti-vacuity: accepting an absent source would delete the only carrier.""" + spool_root = tmp_path / "legacy-hooks" + spool_root.mkdir() + envelope = _hook_envelope("orphan-event") + scratch = tmp_path / "scratch.json" + scratch.write_text(json.dumps(envelope, sort_keys=True), encoding="utf-8") + store = BlobStore(tmp_path / "blob") + _publish_blob(store, _stored_envelope_bytes(scratch)) + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.RESTORE_REQUIRED + assert member.restoration is not None + assert member.restoration.destination is RestorationDestination.HOOK_EVENT_SPOOL + assert member.restoration.logical_id == "orphan-event" + + +def test_same_event_id_with_different_content_is_not_a_source_proof(tmp_path: Path) -> None: + """Anti-vacuity: matching on identity alone would discard divergent material.""" + spool_root = tmp_path / "legacy-hooks" + _write_spool_file(spool_root, _hook_envelope(text="a completely different tool call")) + scratch = tmp_path / "scratch.json" + scratch.write_text(json.dumps(_hook_envelope(text="the stored call"), sort_keys=True), encoding="utf-8") + store = BlobStore(tmp_path / "blob") + _publish_blob(store, _stored_envelope_bytes(scratch)) + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.RESTORE_REQUIRED + + +def test_unclassifiable_material_is_unresolved_and_blocks_acceptance(tmp_path: Path) -> None: + """Anti-vacuity: routing unknown material to discard makes this green wrongly.""" + store = BlobStore(tmp_path / "blob") + _publish_blob(store, b"%PDF-1.5\nnot a session and not an envelope\n") + + context = _context(tmp_path) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + (member,) = plan.members + assert member.disposition is BlobDisposition.UNRESOLVED + assert plan.unresolved_count == 1 + assert not plan.accepted + + +def test_source_file_proof_requires_a_fresh_hash_not_path_existence(tmp_path: Path) -> None: + """Anti-vacuity: proving by path existence accepts a rewritten source.""" + source = tmp_path / "session.jsonl" + source.write_text('{"a": 1}\n', encoding="utf-8") + store = BlobStore(tmp_path / "blob") + blob_hash = _publish_blob(store, source.read_bytes()) + + prover = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source)),)}) + proof = prover.prove(blob_hash, store.blob_path(blob_hash), store.blob_path(blob_hash).stat().st_size) + assert proof is not None and proof.mode is SourceProofMode.BYTE_IDENTICAL + + source.write_text('{"a": 2}\n', encoding="utf-8") + assert prover.prove(blob_hash, store.blob_path(blob_hash), store.blob_path(blob_hash).stat().st_size) is None + + +def test_source_file_proof_accepts_an_exact_append_prefix(tmp_path: Path) -> None: + """Anti-vacuity: requiring whole-file equality would restore every append source.""" + store = BlobStore(tmp_path / "blob") + blob_hash = _publish_blob(store, b'{"a": 1}\n') + source = tmp_path / "session.jsonl" + source.write_bytes(b'{"a": 1}\n{"a": 2}\n') + + prover = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source)),)}) + proof = prover.prove(blob_hash, store.blob_path(blob_hash), 9) + assert proof is not None and proof.mode is SourceProofMode.STRICT_PREFIX + + source.write_bytes(b'{"z": 9}\n{"a": 2}\n') + assert prover.prove(blob_hash, store.blob_path(blob_hash), 9) is None + + +def test_append_prefix_only_supersedes_within_one_logical_item(tmp_path: Path) -> None: + """Anti-vacuity: an unscoped prefix search discards unrelated carriers.""" + store = BlobStore(tmp_path / "blob") + short = _publish_blob(store, b'{"a": 1}\n') + long = _publish_blob(store, b'{"a": 1}\n{"a": 2}\n') + + related = AppendPrefixProver({short: (long,)}, blob_store=store) + assert related.prove(short, store.blob_path(short), 9) is not None + + unrelated = AppendPrefixProver({}, blob_store=store) + assert unrelated.prove(short, store.blob_path(short), 9) is None + + +def test_append_successors_group_by_logical_identity(tmp_path: Path) -> None: + db = _empty_source_db(tmp_path / "source.db") + with sqlite3.connect(db) as conn: + conn.executemany( + "INSERT INTO raw_sessions (raw_id, origin, native_id, blob_hash, blob_size, source_path, " + "append_start_offset) VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + ("r1", "claude-code-session", "s1", bytes.fromhex("aa" * 32), 10, "/tmp/a", None), + ("r2", "claude-code-session", "s1", bytes.fromhex("bb" * 32), 20, "/tmp/a", None), + ("r3", "claude-code-session", "s2", bytes.fromhex("cc" * 32), 30, "/tmp/b", None), + ], + ) + successors = append_successors_by_hash(db) + assert successors == {"aa" * 32: ("bb" * 32,)} + assert raw_source_carriers_by_hash(db)["aa" * 32] == (RawSourceCarrier("/tmp/a"),) + + +def test_reference_union_covers_every_durable_relation(tmp_path: Path) -> None: + """Anti-vacuity: omitting one relation reports its blobs as unreferenced.""" + db = tmp_path / "source.db" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB)") + conn.execute("CREATE TABLE raw_sessions (blob_hash BLOB)") + conn.execute("CREATE TABLE raw_hook_events (blob_hash BLOB)") + conn.execute("CREATE TABLE raw_artifacts (blob_hash BLOB)") + conn.execute("CREATE TABLE blob_publication_reservations (blob_hash BLOB)") + for index, table in enumerate( + ("blob_refs", "raw_sessions", "raw_hook_events", "raw_artifacts", "blob_publication_reservations") + ): + conn.execute(f"INSERT INTO {table} (blob_hash) VALUES (?)", (bytes([index]) * 32,)) + + hashes = referenced_blob_hashes(db) + assert hashes == {bytes([index] * 32).hex() for index in range(5)} + + +def test_unreadable_reference_relation_fails_instead_of_reporting_zero(tmp_path: Path) -> None: + """Anti-vacuity: swallowing the error would license deleting the namespace.""" + db = tmp_path / "source.db" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE blob_refs (blob_hash BLOB)") + conn.execute("CREATE VIEW raw_sessions AS SELECT blob_hash FROM missing_table") + + with pytest.raises(BlobDispositionError): + referenced_blob_hashes(db) + + +def test_plan_digest_binds_denominator_and_every_member(tmp_path: Path) -> None: + """Anti-vacuity: a digest over counts alone lets a member be swapped.""" + store = BlobStore(tmp_path / "blob") + _publish_blob(store, b"%PDF-1.5\nunexplained\n") + context = _context(tmp_path) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + reloaded = BlobDispositionPlan.from_dict(json.loads(json.dumps(plan.to_dict()))) + assert reloaded.digest() == plan.digest() + + mutated = BlobDispositionPlan.from_dict( + { + **plan.to_dict(), + "members": [{**plan.members[0].to_dict(), "disposition": BlobDisposition.SOURCE_PRESENT.value}], + } + ) + assert mutated.digest() != plan.digest() + + +def test_invalid_namespace_entries_block_acceptance(tmp_path: Path) -> None: + """Anti-vacuity: ignoring stray namespace entries hides unaccounted files.""" + blob_root = tmp_path / "blob" + (blob_root / "not-a-shard").mkdir(parents=True) + (blob_root / "not-a-shard" / "stray").write_bytes(b"x") + context = _context(tmp_path) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=blob_root, source_db=tmp_path / "source.db", context=context + ) + + assert plan.denominator.invalid_namespace_entries + assert not plan.accepted + + +def test_denominator_counts_the_complete_population(tmp_path: Path) -> None: + """Anti-vacuity: a sampled census would not reconcile against the walk.""" + spool_root = tmp_path / "legacy-hooks" + spool_file = _write_spool_file(spool_root, _hook_envelope("counted")) + store = BlobStore(tmp_path / "blob") + _publish_blob(store, _stored_envelope_bytes(spool_file)) + _publish_blob(store, b"%PDF-1.5\nunexplained\n") + + context = _context(tmp_path, hook_roots=(("legacy-hook-spool-0", spool_root),)) + plan = compile_disposition_plan( + archive_root=tmp_path, blob_root=store.root, source_db=tmp_path / "source.db", context=context + ) + + assert plan.denominator.physical_file_count == 2 + assert plan.denominator.distinct_hash_count == 2 + assert sum(plan.counts.values()) == 2 + assert plan.counts[BlobDisposition.SOURCE_PRESENT.value] == 1 + assert plan.counts[BlobDisposition.UNRESOLVED.value] == 1 + + +def test_source_file_proof_accepts_the_recorded_append_span(tmp_path: Path) -> None: + """Anti-vacuity: without the recorded span, every increment-only row restores. + + An append-structured acquisition stores just its own increment, so the + object is neither the file nor the file's prefix; only ``file[start:]`` + reproduces it. + """ + store = BlobStore(tmp_path / "blob") + increment = b'{"a": 2}\n' + blob_hash = _publish_blob(store, increment) + source = tmp_path / "session.jsonl" + source.write_bytes(b'{"a": 1}\n' + increment) + + without_span = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source)),)}) + assert without_span.prove(blob_hash, store.blob_path(blob_hash), len(increment)) is None + + with_span = RawSourceFileProver({blob_hash: (RawSourceCarrier(str(source), 9),)}) + proof = with_span.prove(blob_hash, store.blob_path(blob_hash), len(increment)) + assert proof is not None and proof.mode is SourceProofMode.STRICT_PREFIX From 92ec8797558cb0fa3aca770c3f39547c785f16c7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 19:29:52 +0200 Subject: [PATCH 35/47] fix: Own AI Studio attachments on same-timestamp document turns A run of id-less, text-less AI Studio turns sharing one timestamp is separated only by the Drive file its document block cites. Pin that contract on the corpus shape, collapse the block-reference owner tier that the semantic content payload now subsumes, and restore the three identity tests that the partition change left asserting retired behavior. Co-Authored-By: Claude Opus 5 --- polylogue/pipeline/ids.py | 39 +++--------- ...test_message_identity_position_fallback.py | 19 +----- tests/unit/pipeline/test_pipeline_ids.py | 21 ++++--- tests/unit/sources/test_parsers_drive.py | 60 +++++++++++++++++-- 4 files changed, 78 insertions(+), 61 deletions(-) diff --git a/polylogue/pipeline/ids.py b/polylogue/pipeline/ids.py index 75028ee520..b2688e0d90 100644 --- a/polylogue/pipeline/ids.py +++ b/polylogue/pipeline/ids.py @@ -402,28 +402,6 @@ def _message_payload(message: ParsedMessage, fields: frozenset[str]) -> dict[str return payload -def _message_reference_payload(message: ParsedMessage) -> dict[str, JSONValue]: - """Extend the content payload with the reference identity of media blocks. - - An ``image``/``document`` block's metadata names what the turn cites (a - Drive file id, an asset pointer, an inline-content digest). Two id-less, - timestamp-less, text-less turns that cite different files are different - turns, but ``_content_block_payload`` deliberately keeps metadata out of - the session content hash, so this payload is a private owner - discriminator only: it decides ownership after the content payload has - already collided and never feeds a stored hash. - """ - payload = _message_comparison_payload(message) - references: list[JSONValue] = [ - hash_payload(_normalize_nested_for_hash(dict(block.metadata))) - for block in message.blocks - if block.type in (BlockType.IMAGE, BlockType.DOCUMENT) and block.metadata - ] - if references: - payload["block_references"] = references - return payload - - def _message_semantic_payload(message: ParsedMessage) -> dict[str, JSONValue]: """Build the complete semantic payload used by session content hashing.""" return _message_payload(message, _HASHED_FIELDS["ParsedMessage"]) @@ -519,6 +497,13 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol reorder-stable evidence when their content is identical. If neither distinguishes the occurrences, the duplicate remains typed ambiguity instead of receiving a position-derived identity. + + The content discriminator covers a media block's ``metadata`` -- what an + ``image``/``document`` turn cites (a Drive file id, an asset pointer, an + inline-content digest). AI Studio exports carry runs of id-less, + text-less turns that share one timestamp and differ only there, so + ``metadata`` is the sole evidence keeping their attachments ownable + (polylogue-prjai). """ revision_ids = tuple(_message_revision_match_id(message) for message in messages) revision_counts = Counter(revision_ids) @@ -526,25 +511,17 @@ def message_owner_resolution(messages: list[ParsedMessage]) -> MessageOwnerResol f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_comparison_payload(message))}" for message in messages ) content_counts = Counter(content_ids) - reference_ids = tuple( - f"{_CONTENT_ANCHOR_PREFIX}:{hash_payload(_message_reference_payload(message))}" for message in messages - ) - reference_counts = Counter(reference_ids) coordinates = tuple(_message_owner_coordinate(message, index) for index, message in enumerate(messages)) stable_counts = Counter(coordinate.stable_key for coordinate in coordinates if coordinate.stable_key is not None) keys: list[str] = [] - for revision_id, content_id, reference_id, coordinate in zip( - revision_ids, content_ids, reference_ids, coordinates, strict=True - ): + for revision_id, content_id, coordinate in zip(revision_ids, content_ids, coordinates, strict=True): if coordinate.stable_key is not None and stable_counts[coordinate.stable_key] == 1: key = coordinate.stable_key elif revision_counts[revision_id] == 1: key = revision_id elif content_counts[content_id] == 1: key = content_id - elif reference_counts[reference_id] == 1: - key = reference_id elif coordinate.stable_key is not None: key = coordinate.stable_key else: diff --git a/tests/unit/pipeline/test_message_identity_position_fallback.py b/tests/unit/pipeline/test_message_identity_position_fallback.py index a995a4101c..7d3404fa93 100644 --- a/tests/unit/pipeline/test_message_identity_position_fallback.py +++ b/tests/unit/pipeline/test_message_identity_position_fallback.py @@ -291,8 +291,8 @@ def _document_attachment(reference_id: str, position: int) -> ParsedAttachment: def test_duplicate_idless_turns_distinguished_by_referenced_document_own_their_attachments() -> None: """Block reference identity is content: turns citing different documents are different turns. - Anti-vacuity: dropping the block-reference discriminator from - ``message_owner_resolution`` makes both attachments raise + Anti-vacuity: dropping ``metadata`` from + ``_HASHED_FIELDS["ParsedContentBlock"]`` makes both attachments raise ``MessageOwnerAmbiguityError`` here, because the turns share role, timestamp, text, and block type. """ @@ -317,21 +317,6 @@ def test_duplicate_idless_turns_with_identical_document_reference_still_fail_clo session_revision_projection(_session(messages, [attachment])) -def test_block_reference_discriminator_leaves_unique_content_keys_unchanged() -> None: - """The reference tier only runs after the content tier collides.""" - timestamp = "2024-01-01T00:00:00Z" - first_references = [ - _document_only("doc-a", 0).model_copy(update={"text": "first", "timestamp": timestamp}), - _document_only("doc-b", 1).model_copy(update={"text": "second", "timestamp": timestamp}), - ] - other_references = [ - _document_only("doc-x", 0).model_copy(update={"text": "first", "timestamp": timestamp}), - _document_only("doc-y", 1).model_copy(update={"text": "second", "timestamp": timestamp}), - ] - - assert message_owner_resolution(first_references).keys == message_owner_resolution(other_references).keys - - def test_session_content_hash_degrades_ambiguous_attachment_to_unowned() -> None: """A parseable session must survive an attachment owner ambiguity. diff --git a/tests/unit/pipeline/test_pipeline_ids.py b/tests/unit/pipeline/test_pipeline_ids.py index d9e5a1f724..8f08c47795 100644 --- a/tests/unit/pipeline/test_pipeline_ids.py +++ b/tests/unit/pipeline/test_pipeline_ids.py @@ -251,7 +251,12 @@ def test_semantic_hash_partition_rejects_unclassified_and_duplicate_fields(monke def test_duplicate_idless_messages_with_only_position_difference_keep_owner_ambiguous() -> None: - """Position must not turn indistinguishable owner evidence into identity.""" + """Position must not turn indistinguishable owner evidence into identity. + + The strict revision projection is where ownership stays fail-closed; the + ingest content hash instead degrades the attachment to unowned so a + parseable session survives. + """ messages = [ _parsed_message("", "assistant", "repeat", "2024-01-01T00:00:00Z").model_copy(update={"position": position}) for position in (0, 1) @@ -264,12 +269,12 @@ def test_duplicate_idless_messages_with_only_position_difference_keep_owner_ambi mime_type="text/plain", ) + session = _parsed_session("s1", "title", messages, created_at=None, updated_at=None).model_copy( + update={"attachments": [attachment]} + ) + with pytest.raises(MessageOwnerAmbiguityError): - session_content_hash( - _parsed_session("s1", "title", messages, created_at=None, updated_at=None).model_copy( - update={"attachments": [attachment]} - ) - ) + session_revision_projection(session) @pytest.mark.parametrize( @@ -404,10 +409,10 @@ def test_session_revision_projection_golden_hashes() -> None: session = _golden_session() projection = session_revision_projection(session) - assert projection.session_hash.hex() == "87702d4073fdae9932d9b61f4399daa841c77528969645d29b8ac3d6b1134419" + assert projection.session_hash.hex() == "23d0b219777cf59e1b3b8fbe0a16f217e1f8129f9781c2dc4643e665102c4df7" assert [h.hex() for h in projection.message_hashes] == [ "bf3267d2bbb5b9f281401ca940a5a0f339174e750f6dd7b7a5aa70014b00640b", - "a7d0e29040820c1b0285c284a92aa1aad06aca56697aef3b45869f6a31a9bbf3", + "2518ce27da65142108dc3d78e65d7d360202814b6420d8644f50cff3cfe503c1", ] # Content-derived identity (message_id, name, mime_type) -- no longer a # hash of the provider attachment id (polylogue-aggz / polylogue-d8al): diff --git a/tests/unit/sources/test_parsers_drive.py b/tests/unit/sources/test_parsers_drive.py index 1e13e1f464..2da3772b72 100644 --- a/tests/unit/sources/test_parsers_drive.py +++ b/tests/unit/sources/test_parsers_drive.py @@ -802,12 +802,12 @@ def test_idless_document_only_turns_referencing_distinct_files_write_every_attac """AI Studio exports carry id-less, timestamp-less, text-less user turns whose only content is a Drive reference, one turn per file, all files sharing one display name. The turns differ only in the referenced file id, so the - attachment owner resolution must read block reference identity or every - such session fails parse (polylogue-prjai / polylogue-gmb3o). + attachment owner resolution must read the document block's ``metadata`` + or every such session fails parse (polylogue-prjai / polylogue-gmb3o). - Anti-vacuity: removing the block-reference tier from - ``message_owner_resolution`` makes ``session_content_hash`` raise - ``MessageOwnerAmbiguityError`` for this payload. + Anti-vacuity: dropping ``metadata`` from + ``_HASHED_FIELDS["ParsedContentBlock"]`` makes ``session_content_hash`` + raise ``MessageOwnerAmbiguityError`` for this payload. """ payload: JSONDocument = { "chunkedPrompt": { @@ -877,3 +877,53 @@ def inline_image(raw: bytes) -> JSONDocument: rows = conn.execute("SELECT message_id FROM attachment_refs").fetchall() assert len(rows) == 2 assert len({message_id for (message_id,) in rows}) == 2 + + +def test_same_timestamp_document_only_turns_keep_every_attachment_owned( + workspace_env: Mapping[str, Path], +) -> None: + """AI Studio stamps a run of id-less, text-less document turns with one shared + timestamp. Their role/timestamp revision anchor collides by construction, so + the cited Drive file is the only evidence separating them; without it the + attachment owner resolves onto an ambiguous key and the whole session fails + transform with ``attachment owner coordinate is indistinguishable from + another message`` (polylogue-prjai). + + Anti-vacuity: dropping ``metadata`` from + ``_HASHED_FIELDS["ParsedContentBlock"]`` makes + ``session_revision_projection`` raise ``MessageOwnerAmbiguityError`` at + coordinate ``(1, 0)`` and leaves every attachment unowned. + """ + stamp = "2026-04-07T02:20:20.469Z" + payload: JSONDocument = { + "chunkedPrompt": { + "chunks": [ + {"role": "user", "text": "Guidelines follow.", "createTime": "2026-04-06T19:29:05.295Z"}, + {"role": "user", "driveDocument": {"id": "file-a", "name": "chapter.md"}, "createTime": stamp}, + {"role": "user", "driveDocument": {"id": "file-b", "name": "chapter.md"}, "createTime": stamp}, + {"role": "user", "driveDocument": {"id": "file-c", "name": "chapter.md"}, "createTime": stamp}, + {"role": "user", "driveDocument": {"id": "file-d", "name": "chapter.md"}, "createTime": stamp}, + {"role": "model", "text": "Read.", "finishReason": "STOP", "createTime": stamp}, + ] + } + } + + result = parse_chunked_prompt("gemini", payload, "gemini-same-timestamp-documents") + assert [message.provider_message_id for message in result.messages] == [""] * 6 + assert {message.timestamp for message in result.messages[1:]} == {stamp} + assert [attachment.provider_attachment_id for attachment in result.attachments] == [ + "file-a", + "file-b", + "file-c", + "file-d", + ] + + # The strict projection is the path that has no ambiguous-owner tolerance. + session_revision_projection(result) + + db_path = db_setup(workspace_env) + with open_connection(db_path) as conn: + write_and_hydrate(PipelineRoundtrip(result, session_content_hash(result)), conn) + rows = conn.execute("SELECT message_id, attachment_id FROM attachment_refs ORDER BY message_id").fetchall() + assert len(rows) == 4 + assert len({message_id for message_id, _attachment_id in rows}) == 4 From 10769ce8e3b88964b5317b35046dab47d9cf877a Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 20:29:48 +0200 Subject: [PATCH 36/47] fix: Apply the conversational-evidence law in the one-shot importer `require_positive_conversational_evidence` is the archive's admission law for "parsed, but no conversation is present". The daemon decode worker, live batch convergence, the incremental append route, and offline replay apply it; `pipeline/services/archive_ingest.py::parse_sources_archive`, the importer behind the public `parse_file`/`parse_sources` API and the demo seeder, did not. A JSON document under a watched Claude Code project that satisfies only dispatch's loose messages-list shape was therefore written as a session keyed on its own filename stem with zero authored messages -- the fragment-identity phantom class, arriving through the one chokepoint that did not agree with the other four. Also pins three laws that had no anti-vacuity coverage: a `tool-results/` sidecar never becomes a session on either chokepoint whatever it contains; the importer refuses `toolu_*`/`wf_*` stem identity while still admitting a transcript with content; a sidecar event with no readable file mtime keeps its time unknown instead of inventing an ingestion-time stamp. Co-Authored-By: Claude Opus 5 --- polylogue/pipeline/services/archive_ingest.py | 15 ++++ polylogue/sources/dispatch.py | 6 +- .../test_archive_ingest_shared_raw.py | 79 +++++++++++++++++ .../unit/sources/test_tool_result_sidecars.py | 88 ++++++++++++++++++- 4 files changed, 185 insertions(+), 3 deletions(-) diff --git a/polylogue/pipeline/services/archive_ingest.py b/polylogue/pipeline/services/archive_ingest.py index 10b75d84de..2893d71bd9 100644 --- a/polylogue/pipeline/services/archive_ingest.py +++ b/polylogue/pipeline/services/archive_ingest.py @@ -26,6 +26,7 @@ open_bounded_zip_entry, zip_entry_session_artifact, ) +from polylogue.sources.dispatch import require_positive_conversational_evidence from polylogue.sources.parsers import antigravity from polylogue.sources.parsers.base import ParsedSession, RawSessionData from polylogue.sources.source_parsing import ( @@ -158,6 +159,20 @@ async def write_pair( raw_data: RawSessionData | None, session: ParsedSession, ) -> None: + # polylogue-b508: a session requires positive evidence of a + # conversation. The daemon decode worker, live batch convergence, + # the incremental append route, and offline replay each apply this + # law right after dispatch returns; this one-shot importer is the + # remaining production write path and must agree, or a document + # that merely satisfies dispatch's loose messages-list shape is + # written as a session keyed on its own filename stem -- identity + # the discovery walk invented, not identity a provider asserted. + if not require_positive_conversational_evidence( + [session], + provider=session.source_name, + source_path=_archive_raw_source_path(raw_data, source), + ): + return session = normalize_session_timestamps( session, fallback_timestamp=raw_data.file_mtime if raw_data is not None else None, diff --git a/polylogue/sources/dispatch.py b/polylogue/sources/dispatch.py index c560d092a3..3fce07ff1c 100644 --- a/polylogue/sources/dispatch.py +++ b/polylogue/sources/dispatch.py @@ -1693,11 +1693,13 @@ def require_positive_conversational_evidence( which already treats an empty session list as a recorded, bounded ``mark_raw_parse_failed`` outcome -- this filter reuses that existing "refused loudly" mechanism rather than inventing a new one), - ``sources/live/append_ingest.py`` (incremental append), and + ``sources/live/append_ingest.py`` (incremental append), ``sources/revision_backfill.py`` (offline replay/rebuild, alongside its own OriginSpec/``classify_artifact`` path-and-shape gate from polylogue-6mpy -- this filter catches the sibling case where the shape - is recognized but the parsed *content* still carries no message). + is recognized but the parsed *content* still carries no message), and + ``pipeline/services/archive_ingest.py`` (the one-shot importer behind + ``Polylogue.parse_file``/``parse_sources`` and the demo seeder). Measured against the live archive (2026-07-31, read-only query against ``index.db``/``source.db``): every verified zero-message diff --git a/tests/unit/pipeline/test_archive_ingest_shared_raw.py b/tests/unit/pipeline/test_archive_ingest_shared_raw.py index 67ee690960..ed5aa31b72 100644 --- a/tests/unit/pipeline/test_archive_ingest_shared_raw.py +++ b/tests/unit/pipeline/test_archive_ingest_shared_raw.py @@ -806,3 +806,82 @@ def require_source_transaction( assert len(_raw_rows_for_path(archive_root / "source.db", str(second_child))) == 1 with sqlite3.connect(f"file:{archive_root / 'index.db'}?mode=ro", uri=True) as conn: assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 4 + + +def _write_stem_identity_husk(root: Path, stem: str) -> Path: + """A JSON document under a Claude Code project whose only identity is its filename. + + The record shape satisfies dispatch's loose "looks like a message list" + admission but carries no record the Claude Code parser recognizes, so the + parse falls back to the discovery walk's ``fallback_id`` (the filename + stem) and yields a session with zero authored messages. + """ + path = root / ".claude" / "projects" / "proj" / "notes" / f"{stem}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({"messages": [{"role": "user", "content": "tool output shaped like a chat"}]}), + encoding="utf-8", + ) + return path + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stem", ["toolu_01ABCDEFGHIJKLMNOPQRSTUV", "wf_run_1"]) +async def test_archive_ingest_refuses_filename_stem_identity_without_authored_content( + tmp_path: Path, workspace_env: dict[str, Path], stem: str +) -> None: + """polylogue-b508: the one-shot importer must not mint fragment-identity husks. + + ``require_positive_conversational_evidence`` is the archive's admission law + for "parsed, but no conversation is present". Every other production write + path applies it -- the daemon decode worker, live batch convergence, the + incremental append route, and offline replay. ``parse_sources_archive`` + (reached from the public ``Polylogue.parse_file``/``parse_sources`` API and + the demo seeder) did not, so a JSON document that merely satisfies the + loose "has a messages list" shape became a session keyed on its own + filename stem with zero messages -- the ``toolu_*``/``wf_*`` fragment + phantom class this bead exists to make unrepresentable. (The ``*.meta`` + sibling shape is refused earlier, by its own declared artifact rule.) + + Anti-vacuity: dropping the ``write_pair`` evidence gate writes one session + row per parametrized stem, each ``provider_session_id`` equal to the stem + and ``COUNT(*) FROM messages`` zero. + """ + archive_root = workspace_env["archive_root"] + husk = _write_stem_identity_husk(tmp_path / "corpus", stem) + + result = await parse_sources_archive( + archive_root, + [Source(name="claude-code", path=husk)], + parse_workers=1, + ) + + assert result.parse_failures == 0 + assert result.counts.get("sessions", 0) == 0 + with sqlite3.connect(f"file:{archive_root / 'index.db'}?mode=ro", uri=True) as conn: + assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 0 + + +@pytest.mark.asyncio +async def test_archive_ingest_still_admits_a_real_session_through_the_same_gate( + tmp_path: Path, workspace_env: dict[str, Path] +) -> None: + """The evidence gate must not refuse a transcript that carries authored content. + + Pairs with the husk law above: the refusal is keyed on absent authored + content, not on the file's location or its identity shape. Anti-vacuity: + widening the gate to reject on anything the husk case has in common with + this one (same directory, same provider, same one-shot route) turns this + red. + """ + archive_root = workspace_env["archive_root"] + transcript = _write_session_shaped_workflow_journal(tmp_path / "sessions") + + result = await parse_sources_archive( + archive_root, + [Source(name="claude-code", path=transcript)], + parse_workers=1, + ) + + assert result.parse_failures == 0 + assert result.counts["sessions"] == 1 diff --git a/tests/unit/sources/test_tool_result_sidecars.py b/tests/unit/sources/test_tool_result_sidecars.py index e32480d6a1..f5ebc8259f 100644 --- a/tests/unit/sources/test_tool_result_sidecars.py +++ b/tests/unit/sources/test_tool_result_sidecars.py @@ -12,7 +12,8 @@ import os from pathlib import Path -from polylogue.core.enums import BlockType +from polylogue.config import Source +from polylogue.core.enums import BlockType, Provider from polylogue.sources.live.tool_result_sidecars import ( SidecarDebt, SidecarMatch, @@ -20,7 +21,10 @@ join_tool_result_sidecars_session_scoped, resolve_sibling_transcript_paths, ) +from polylogue.sources.origin_specs import artifact_rule_for_path from polylogue.sources.parsers.claude.code_parser import apply_tool_result_sidecars, parse_code +from polylogue.sources.revision_backfill import _parse_one +from polylogue.sources.source_parsing import iter_source_sessions_with_raw _TRUNCATED_NEEDLE = "zz_sentinel_needle_only_in_full_output" @@ -299,3 +303,85 @@ def test_session_scoped_join_never_emits_debt_for_subagent_meta_companion_files( assert result.matched == () assert result.debt == () + + +def test_tool_results_sidecar_never_becomes_a_session_on_either_chokepoint(tmp_path: Path) -> None: + """polylogue-b508: a ``tool-results/`` file is raw-only, whatever it contains. + + A tool call's own output can reproduce a genuine session-document shape -- + a messages list, even an ``id`` field shaped like the ``toolu_*`` id the + sidecar is named for. Content heuristics alone therefore cannot refuse this + family; the ``tool_result_sidecar`` path rule is the gate, and both parse + chokepoints must honour it: the discovery/acquisition walk that the one-shot + importer and the live watcher share, and the offline replay engine that + rebuilds from retained raws. + + Anti-vacuity: widening ``tool_result_sidecar``'s ``path_pattern`` so it no + longer matches, or relaxing its ``parse_policy`` from ``raw-only``, admits + a session whose ``provider_session_id`` is the ``toolu_*`` fragment id -- + the phantom shape this law forbids. + """ + body = json.dumps( + { + "id": "toolu_01ABCDEFGHIJKLMNOPQRSTUV", + "messages": [ + {"role": "user", "content": "tool output that happens to look like a chat"}, + {"role": "assistant", "content": "reply"}, + ], + } + ).encode("utf-8") + + sidecar = ( + tmp_path / ".claude" / "projects" / "proj" / "sess" / "tool-results" / "toolu_01ABCDEFGHIJKLMNOPQRSTUV.json" + ) + sidecar.parent.mkdir(parents=True) + sidecar.write_bytes(body) + + rule = artifact_rule_for_path(Provider.CLAUDE_CODE, str(sidecar)) + assert rule is not None + assert (rule.kind, rule.parse_policy) == ("tool_result_sidecar", "raw-only") + + replayed = _parse_one(Provider.CLAUDE_CODE, body, str(sidecar)) + assert replayed == [] + + acquired = list(iter_source_sessions_with_raw(Source(name="claude-code", path=sidecar), capture_raw=False)) + assert [session.provider_session_id for _raw, session in acquired] == [] + + +def test_sidecar_event_time_stays_unknown_when_the_file_carries_no_mtime_evidence() -> None: + """polylogue-x1gd: absent time evidence stays typed unknown, never invented. + + The sidecar file's own mtime is the only timestamp evidence this join has -- + sidecars carry no embedded time, and for genuine debt the owning + ``tool_result`` block is by definition unresolvable. When ``stat`` cannot + supply it (``file_mtime_ms`` stays ``None``), the emitted + ``claude_tool_result_sidecar`` event must carry no timestamp, leaving + ``occurred_at_ms`` NULL, rather than an ingestion-time stamp that would + read downstream as "this sidecar was written at import". + + Anti-vacuity: defaulting the missing-mtime branch to the current clock or + to the transcript's own timestamp makes both ``event.timestamp`` values + non-None and turns this red. + """ + from polylogue.sources.live.tool_result_sidecars import SidecarJoinResult + + payload = [_record("m-aaa", "toolu_AAA", "small full text")] + join_result = SidecarJoinResult( + matched=( + SidecarMatch( + tool_use_id="toolu_AAA", + filename="toolu_AAA.txt", + byte_size=15, + content_hash="0" * 64, + was_truncated=False, + full_text="small full text", + ), + ), + debt=(SidecarDebt(filename="orphan123.txt", byte_size=7, reason="no_owning_tool_result_block"),), + ) + + acquired = parse_code(payload, "fallback-sidecar", tool_result_sidecars=join_result) + + sidecar_events = [event for event in acquired.session_events if event.event_type == "claude_tool_result_sidecar"] + assert len(sidecar_events) == 2 + assert [event.timestamp for event in sidecar_events] == [None, None] From 642f78d041488a93bc54dc62f3d401b9a1088636 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 20:19:42 +0200 Subject: [PATCH 37/47] test: Cover attachment reacquisition across capture revisions An upload-only claude.ai `files` reference gains its bytes when a later capture revision carries them as `extracted_content`. The bytes must land `acquired` under the same attachment identity: a second identity strands the original reference and double-counts the attachment. Co-Authored-By: Claude Opus 5 --- .../storage/test_attachment_reacquisition.py | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 tests/unit/storage/test_attachment_reacquisition.py diff --git a/tests/unit/storage/test_attachment_reacquisition.py b/tests/unit/storage/test_attachment_reacquisition.py new file mode 100644 index 0000000000..9fc6e9816f --- /dev/null +++ b/tests/unit/storage/test_attachment_reacquisition.py @@ -0,0 +1,134 @@ +"""Attachment reacquisition across capture revisions (polylogue-4zqh3). + +An upload-only claude.ai ``files`` reference records a name and a size but no +bytes, so its first ingest is honestly ``unfetched``. When a later revision of +the same capture carries the payload as ``extracted_content``, the bytes must +land as an ``acquired`` blob under the *same* attachment identity — a second +identity would strand the original reference and double-count the attachment. + +Anti-vacuity: drop ``extracted_content`` from the ``files`` branch of +``attachment_from_meta`` and the second revision stays ``unfetched``; fold +acquisition state into ``_attachment_id`` and the two revisions mint different +identities, growing the reference count. +""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from pathlib import Path + +import pytest + +from polylogue.sources.parsers.base import ParsedSession +from polylogue.sources.parsers.claude import parse_ai +from polylogue.storage.blob_store import BlobStore +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.archive_tiers.write import write_parsed_session_to_archive + +SESSION_UUID = "reacquisition-session" +FILE_UUID = "upload-only-file" +PAYLOAD = "restored attachment payload\n" +PAYLOAD_BYTES = PAYLOAD.encode("utf-8") + + +def _capture(*, extracted_content: str | None) -> dict[str, object]: + """One claude.ai capture revision carrying a single upload-only reference.""" + file_record: dict[str, object] = { + "file_uuid": FILE_UUID, + "uuid": FILE_UUID, + "file_kind": "blob", + "file_name": "restored.md", + "size_bytes": len(PAYLOAD_BYTES), + "path": "/mnt/user-data/uploads/restored.md", + "success": True, + } + if extracted_content is not None: + file_record["extracted_content"] = extracted_content + return { + "uuid": SESSION_UUID, + "name": "Attachment reacquisition", + "chat_messages": [ + { + "uuid": "m0", + "sender": "human", + "text": "Please read this.", + "files": [file_record], + } + ], + } + + +def _connect(path: Path) -> sqlite3.Connection: + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + initialize_archive_tier(conn, ArchiveTier.INDEX) + return conn + + +def _preacquired(store: BlobStore, session: ParsedSession) -> dict[int, tuple[bytes | None, int, str]]: + acquired: dict[int, tuple[bytes | None, int, str]] = {} + for attachment in session.attachments: + if attachment.inline_bytes is None: + continue + blob_hash, size = store.write_from_bytes(attachment.inline_bytes) + acquired[id(attachment)] = (bytes.fromhex(blob_hash), size, "acquired") + return acquired + + +def _attachment_state(conn: sqlite3.Connection) -> sqlite3.Row: + row: sqlite3.Row | None = conn.execute( + "SELECT attachment_id, display_name, byte_count, blob_hash, acquisition_status FROM attachments" + ).fetchone() + assert row is not None + return row + + +def _ref_count(conn: sqlite3.Connection) -> int: + return int(conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0]) + + +def test_upload_only_reference_gains_bytes_at_a_stable_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + store = BlobStore(tmp_path / "blob") + monkeypatch.setattr("polylogue.storage.blob_store.get_blob_store", lambda: store) + conn = _connect(tmp_path / "index.db") + + before = parse_ai(_capture(extracted_content=None), "fallback") + write_parsed_session_to_archive(conn, before, preacquired_attachment_blobs=_preacquired(store, before)) + + unfetched = _attachment_state(conn) + assert unfetched["acquisition_status"] == "unfetched" + assert unfetched["blob_hash"] is None + assert unfetched["byte_count"] == len(PAYLOAD_BYTES) + identity = str(unfetched["attachment_id"]) + assert _ref_count(conn) == 1 + + after = parse_ai(_capture(extracted_content=PAYLOAD), "fallback") + write_parsed_session_to_archive(conn, after, preacquired_attachment_blobs=_preacquired(store, after)) + + acquired = _attachment_state(conn) + assert str(acquired["attachment_id"]) == identity + assert acquired["acquisition_status"] == "acquired" + assert bytes(acquired["blob_hash"]) == hashlib.sha256(PAYLOAD_BYTES).digest() + assert acquired["byte_count"] == len(PAYLOAD_BYTES) + assert store.read_all(hashlib.sha256(PAYLOAD_BYTES).hexdigest()) == PAYLOAD_BYTES + assert _ref_count(conn) == 1 + + +def test_replaying_the_acquired_revision_changes_nothing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + store = BlobStore(tmp_path / "blob") + monkeypatch.setattr("polylogue.storage.blob_store.get_blob_store", lambda: store) + conn = _connect(tmp_path / "index.db") + + for _ in range(2): + session = parse_ai(_capture(extracted_content=PAYLOAD), "fallback") + write_parsed_session_to_archive(conn, session, preacquired_attachment_blobs=_preacquired(store, session)) + + acquired = _attachment_state(conn) + assert acquired["acquisition_status"] == "acquired" + assert bytes(acquired["blob_hash"]) == hashlib.sha256(PAYLOAD_BYTES).digest() + assert _ref_count(conn) == 1 From ad0e32f4abdd65f6b7b56f1d582a79a93204d876 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 20:31:39 +0200 Subject: [PATCH 38/47] perf: Size the archive-ingest parse pool from the walk it found resolve_archive_ingest_dispatch read CPU count alone, so every parse_sources_archive call spawned the ambient worker count -- 16 fresh interpreters on a 24-thread host -- even for a walk of one small file, the shape the API ingest facade produces per source. Tier the plan on the byte boundaries resolve_ingest_batch_dispatch already uses, and record the walk, parse, and pool phases in the append.* stage ledger that previously covered only the archive write. Measured over six one-file walks (1,074 messages), interleaved arms: 41.2/41.5 messages/s at 0.078 process CPU utilization with the pool, 325.4/476.3 at 0.795/0.764 in-process. Co-Authored-By: Claude Opus 5 --- polylogue/pipeline/services/archive_ingest.py | 176 +++++++++++++----- polylogue/pipeline/services/process_pool.py | 33 ++-- .../test_archive_ingest_commit_batching.py | 66 ++++++- tests/unit/pipeline/test_process_pool.py | 53 ++++-- 4 files changed, 254 insertions(+), 74 deletions(-) diff --git a/polylogue/pipeline/services/archive_ingest.py b/polylogue/pipeline/services/archive_ingest.py index 2893d71bd9..4ba3876fa0 100644 --- a/polylogue/pipeline/services/archive_ingest.py +++ b/polylogue/pipeline/services/archive_ingest.py @@ -3,9 +3,14 @@ from __future__ import annotations import json +import time import zipfile +from collections.abc import Callable, Mapping from concurrent.futures import as_completed +from contextlib import suppress +from dataclasses import dataclass from datetime import UTC, datetime +from functools import partial from pathlib import Path from typing import Any @@ -17,8 +22,10 @@ from polylogue.logging import get_logger from polylogue.pipeline.services.parsing_models import ParseResult from polylogue.pipeline.services.process_pool import ( + PoolKind, process_pool_executor, resolve_archive_ingest_dispatch, + resolve_parse_worker_count, ) from polylogue.sources.decoder_zip import ( ZipBombError, @@ -58,6 +65,41 @@ POST_COMMIT_UPKEEP_REASON = "archive_ingest_commit" +@dataclass(frozen=True, slots=True) +class _ParseSubmission: + """One walked source file and everything its parse worker needs.""" + + source: Source + path: Path + file_mtime: Any + sidecar_data: Mapping[str, Any] | None + + +def _record_stage(result: ParseResult, name: str, started_at: float) -> None: + """Accumulate one ingest phase into the run's stage ledger. + + Keeps the walk and parse phases inside the same ``append.*`` namespace the + archive write path already reports, so the ledger sums toward wall time + instead of describing the write alone. + """ + key = f"append.{name}" + result.stage_timings_s[key] = result.stage_timings_s.get(key, 0.0) + (time.perf_counter() - started_at) + + +def _submission_payload_bytes(submissions: list[_ParseSubmission]) -> int: + """Total on-disk size of the files this walk will parse. + + Sizes the parse dispatch. A path that vanished between the walk and here + contributes nothing, which biases the plan toward sequential -- the safe + direction, since the pool only pays off above the byte tiers. + """ + total = 0 + for submission in submissions: + with suppress(OSError): + total += submission.path.stat().st_size + return total + + def _commit_batch_message_threshold() -> int: from polylogue.config import load_polylogue_config @@ -114,6 +156,10 @@ async def parse_sources_archive( resolve out-of-order. Blob writes from workers are content-addressed and atomic, so concurrent worker writes are process-safe. + The pool is sized from the work the walk actually found + (:func:`resolve_archive_ingest_dispatch`), so a small walk parses + in-process instead of paying a spawn per worker for it. + ``parse_workers`` overrides the ambient/env-resolved worker count for this call only (used by the demo seeder to force sequential parsing -- see ``polylogue/demo/seed.py``). ``None`` preserves the normal @@ -123,7 +169,7 @@ async def parse_sources_archive( acquired_at_ms = int(datetime.now(UTC).timestamp() * 1000) threshold = _commit_batch_message_threshold() batched = threshold > 0 - workers = resolve_archive_ingest_dispatch(parse_workers=parse_workers).worker_count + workers = resolve_parse_worker_count() if parse_workers is None else max(1, parse_workers) blob_root = archive_root / "blob" from polylogue.storage.blob_publication import ArchiveBlobPublisher @@ -337,53 +383,91 @@ async def write_pair( await write_pair(source, raw_data, session) failed = 0 - total_paths = 0 - with process_pool_executor(max_workers=workers) as pool: - future_to_source: dict[Any, tuple[Source, Path]] = {} - for source in sources: - walk = _setup_source_walk( - source, - cursor_state=None, - include_mtime=True, - known_mtimes=None, - discover_sidecars=True, - blob_store=parse_blob_publisher, - ) - if walk is None: + submissions: list[_ParseSubmission] = [] + walk_started_at = time.perf_counter() + for source in sources: + walk = _setup_source_walk( + source, + cursor_state=None, + include_mtime=True, + known_mtimes=None, + discover_sidecars=True, + blob_store=parse_blob_publisher, + ) + if walk is None: + continue + for path, file_mtime in walk.paths_to_process: + if ( + Provider.from_string(source.name) is Provider.ANTIGRAVITY + and antigravity.classify_source_path(path).role + is antigravity.AntigravitySourceRole.CONVERSATION_PROTOBUF + ): continue - for path, file_mtime in walk.paths_to_process: - if ( - Provider.from_string(source.name) is Provider.ANTIGRAVITY - and antigravity.classify_source_path(path).role - is antigravity.AntigravitySourceRole.CONVERSATION_PROTOBUF - ): - continue - future = pool.submit( - _parse_source_path_worker, - str(path), - file_mtime, - source.name, - walk.sidecar_data, - True, - str(blob_root), - str(archive_root / "source.db"), + submissions.append(_ParseSubmission(source, path, file_mtime, walk.sidecar_data)) + total_paths = len(submissions) + _record_stage(result, "walk", walk_started_at) + + def _parse_args(submission: _ParseSubmission) -> tuple[Any, ...]: + return ( + str(submission.path), + submission.file_mtime, + submission.source.name, + submission.sidecar_data, + True, + str(blob_root), + str(archive_root / "source.db"), + ) + + async def consume(source: Source, path: Path, produce: Callable[[], Any]) -> None: + nonlocal failed + parse_started_at = time.perf_counter() + try: + pairs = produce() + except Exception as exc: + # Worker error isolation: one bad file must not kill the + # run. Mirror the sequential iterator's failure handling. + _record_stage(result, "parse", parse_started_at) + failed += 1 + result.parse_failures += 1 + logger.error("Failed to parse %s in worker: %s", path, exc) + return + _record_stage(result, "parse", parse_started_at) + for raw_data, session in pairs: + await write_pair(source, raw_data, session) + + plan = resolve_archive_ingest_dispatch( + path_count=total_paths, + total_bytes=_submission_payload_bytes(submissions), + worker_ceiling=workers, + ) + if plan.pool_kind is PoolKind.SEQUENTIAL: + for submission in submissions: + await consume( + submission.source, + submission.path, + partial(_parse_source_path_worker, *_parse_args(submission)), + ) + elif submissions: + pool_started_at = time.perf_counter() + with process_pool_executor(max_workers=plan.worker_count) as pool: + future_to_source: dict[Any, tuple[Source, Path]] = { + pool.submit(_parse_source_path_worker, *_parse_args(submission)): ( + submission.source, + submission.path, ) - future_to_source[future] = (source, path) - total_paths += 1 - - for future in as_completed(future_to_source): - source, path = future_to_source[future] - try: - pairs = future.result() - except Exception as exc: - # Worker error isolation: one bad file must not kill the - # run. Mirror the sequential iterator's failure handling. - failed += 1 - result.parse_failures += 1 - logger.error("Failed to parse %s in worker: %s", path, exc) - continue - for raw_data, session in pairs: - await write_pair(source, raw_data, session) + for submission in submissions + } + _record_stage(result, "parse_pool", pool_started_at) + # Workers spawn lazily behind `submit`, so the parse itself + # is the wait between completions, not `future.result()`. + wait_started_at = time.perf_counter() + for future in as_completed(future_to_source): + _record_stage(result, "parse", wait_started_at) + source, path = future_to_source[future] + await consume(source, path, future.result) + wait_started_at = time.perf_counter() + shutdown_started_at = time.perf_counter() + _record_stage(result, "parse_pool", shutdown_started_at) if failed > 0: logger.warning( diff --git a/polylogue/pipeline/services/process_pool.py b/polylogue/pipeline/services/process_pool.py index 82af066770..7795816ed1 100644 --- a/polylogue/pipeline/services/process_pool.py +++ b/polylogue/pipeline/services/process_pool.py @@ -147,19 +147,28 @@ class ParseDispatchPlan: worker_count: int -def resolve_archive_ingest_dispatch(*, parse_workers: int | None = None) -> ParseDispatchPlan: - """Worker-count decision for ``archive_ingest.py``'s re-ingest file-walk parse. - - Unchanged formula: an explicit ``parse_workers`` override (clamped to at - least 1) wins; otherwise :func:`resolve_parse_worker_count` (CPU count, - ceiling adjusted for a free-threaded build). Always a process pool -- the - caller's own ``workers <= 1`` branch is the escape hatch to sequential, - preserved unchanged at the call site rather than folded into - :data:`PoolKind` here, since that branch also skips constructing the pool - context entirely (a real, not merely nominal, sequential path). +def resolve_archive_ingest_dispatch(*, path_count: int, total_bytes: int, worker_ceiling: int) -> ParseDispatchPlan: + """Pool-kind + worker-count decision for ``archive_ingest.py``'s file-walk parse. + + Sized by the work the walk actually found, on the same byte tiers as + :func:`resolve_ingest_batch_dispatch`: ``<= 8 MiB`` sequential, ``<= 64 + MiB`` capped at 4 workers, above that ``min(path_count, cpus, ceiling)``. + A spawn pool costs a fresh interpreter and a full ``polylogue`` import per + worker; below the first tier that setup exceeds the parse it replaces, and + a spawn failure under host pressure is absorbed by the driver's per-file + ``except`` as a silently dropped file rather than surfacing as an error. + + ``worker_ceiling`` is the caller's already-resolved + :func:`resolve_parse_worker_count` value, so the operator knob keeps one + home. A ceiling of 1 never reaches here: it selects the caller's + source-iterator escape hatch, which is a different route from the walk. """ - worker_count = resolve_parse_worker_count() if parse_workers is None else max(1, parse_workers) - return ParseDispatchPlan(PoolKind.PROCESS, worker_count) + if path_count <= 1 or total_bytes <= 8 * 1024 * 1024: + return ParseDispatchPlan(PoolKind.SEQUENTIAL, 1) + cpus = available_cpus() or 4 + if total_bytes <= 64 * 1024 * 1024: + return ParseDispatchPlan(PoolKind.PROCESS, max(1, min(path_count, cpus, worker_ceiling, 4))) + return ParseDispatchPlan(PoolKind.PROCESS, max(1, min(path_count, cpus, worker_ceiling))) def resolve_validation_dispatch(*, record_count: int) -> ParseDispatchPlan: diff --git a/tests/unit/pipeline/test_archive_ingest_commit_batching.py b/tests/unit/pipeline/test_archive_ingest_commit_batching.py index ce142bbae6..1d37a2777b 100644 --- a/tests/unit/pipeline/test_archive_ingest_commit_batching.py +++ b/tests/unit/pipeline/test_archive_ingest_commit_batching.py @@ -506,6 +506,10 @@ def fake_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: # min(8, cpus-1), which is >= 2 on any real multi-core CI/dev host, so # an un-overridden call below would exercise the pool branch. monkeypatch.delenv("POLYLOGUE_INGEST_PARSE_WORKERS", raising=False) + # The walk-size tiering would send this synthetic corpus down the + # in-process branch on bytes alone; report a bulk-sized walk so the + # override, not the tier, is what this test measures. + monkeypatch.setattr(archive_ingest, "_submission_payload_bytes", lambda _submissions: 128 * 1024 * 1024) archive_root = workspace_env["archive_root"] sequential_sources = _build_sources(tmp_path, count=2, seed=97) @@ -513,7 +517,7 @@ def fake_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: assert pool_calls == [] # workers<=1 takes the sequential for-loop, no pool constructed assert result.counts["sessions"] == _expected_session_count(sequential_sources) - pooled_sources = _build_sources(tmp_path, count=2, seed=98) + pooled_sources = _build_sources(tmp_path, count=4, seed=98) result = asyncio.run(parse_sources_archive(archive_root, pooled_sources, parse_workers=3)) assert pool_calls == [3] # explicit override reaches the pool construction exactly assert result.counts["sessions"] == _expected_session_count(pooled_sources) @@ -554,6 +558,9 @@ def spying_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: monkeypatch.setattr(archive_ingest, "process_pool_executor", spying_process_pool_executor) monkeypatch.delenv("POLYLOGUE_INGEST_PARSE_WORKERS", raising=False) + # Report a bulk-sized walk so the dispatch tiering selects the pool this + # test exists to inspect (see _submission_payload_bytes). + monkeypatch.setattr(archive_ingest, "_submission_payload_bytes", lambda _submissions: 128 * 1024 * 1024) archive_root = workspace_env["archive_root"] sources = _build_sources(tmp_path, count=2, seed=113) @@ -562,3 +569,60 @@ def spying_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: assert calls == [2] assert result.counts["sessions"] == _expected_session_count(sources) assert not hasattr(archive_ingest, "ProcessPoolExecutor") + + +def _walk_bytes(sources: Sequence[Source]) -> int: + return sum(source.path.stat().st_size for source in sources if source.path is not None) + + +def test_small_walk_parses_in_process_with_identical_results( + tmp_path: Path, + workspace_env: dict[str, Path], + empty_archive_template: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An ordinary small walk parses in-process and lands exactly what the pool lands. + + ``resolve_archive_ingest_dispatch`` used to read CPU count alone, so a walk + of a handful of small files spawned one fresh interpreter per ambient + worker -- a full ``polylogue`` import each -- to parse them. Measured on a + 24-thread host, six one-file walks went 33-41 messages/s with the pool and + 401-424 messages/s without it, at 0.06 versus 0.84 process CPU utilization. + + Anti-vacuity, two ways: reverting the tiering makes the first arm construct + a pool and fail on ``pool_calls``; routing the in-process branch through + anything other than the same ``_parse_source_path_worker`` call makes the + two arms' session and message counts diverge. + """ + from polylogue.pipeline.services.process_pool import process_pool_executor as real_process_pool_executor + from tests.infra.archive_templates import clone_archive_template + + pool_calls: list[int] = [] + + def spying_process_pool_executor(*, max_workers: int) -> ProcessPoolExecutor: + pool_calls.append(max_workers) + return real_process_pool_executor(max_workers=max_workers) + + monkeypatch.setattr(archive_ingest, "process_pool_executor", spying_process_pool_executor) + monkeypatch.delenv("POLYLOGUE_INGEST_PARSE_WORKERS", raising=False) + + in_process_root = workspace_env["archive_root"] + sources = _build_sources(tmp_path, count=3, seed=211) + assert _walk_bytes(sources) <= 8 * 1024 * 1024 # the tier the first arm relies on + + in_process = asyncio.run(parse_sources_archive(in_process_root, sources)) + assert pool_calls == [] + assert in_process.counts["sessions"] == _expected_session_count(sources) + + pooled_root = tmp_path / "pooled-archive" + clone_archive_template(empty_archive_template, pooled_root) + monkeypatch.setattr(archive_ingest, "_submission_payload_bytes", lambda _submissions: 128 * 1024 * 1024) + pooled = asyncio.run(parse_sources_archive(pooled_root, sources)) + # One pool, sized by min(path_count, cpus, ceiling) -- bounded by the walk + # rather than fixed, so this holds on a narrower host too. + assert len(pool_calls) == 1 + assert 2 <= pool_calls[0] <= len(sources) + + assert _counts(pooled_root / "index.db") == _counts(in_process_root / "index.db") + assert pooled.counts["sessions"] == in_process.counts["sessions"] + assert pooled.counts["messages"] == in_process.counts["messages"] diff --git a/tests/unit/pipeline/test_process_pool.py b/tests/unit/pipeline/test_process_pool.py index c99148ae7c..00f48482a2 100644 --- a/tests/unit/pipeline/test_process_pool.py +++ b/tests/unit/pipeline/test_process_pool.py @@ -171,26 +171,49 @@ def test_parallel_threads_effective_treats_missing_probe_as_gil_enabled(monkeypa # failing first. -def test_resolve_archive_ingest_dispatch_defaults_to_resolve_parse_worker_count( +@pytest.mark.parametrize( + ("path_count", "total_bytes", "cpu_count", "expected_kind", "expected_workers"), + [ + # A single path never justifies a spawn, whatever it weighs. + (1, 512 * 1024 * 1024, 24, PoolKind.SEQUENTIAL, 1), + # Small-byte tier: in-process, matching resolve_ingest_batch_dispatch. + (400, 8 * 1024 * 1024, 24, PoolKind.SEQUENTIAL, 1), + # Mid tier: capped at 4 workers. + (400, 8 * 1024 * 1024 + 1, 24, PoolKind.PROCESS, 4), + (400, 64 * 1024 * 1024, 24, PoolKind.PROCESS, 4), + # Above the mid tier: min(path_count, cpus, ceiling). + (400, 64 * 1024 * 1024 + 1, 24, PoolKind.PROCESS, 16), + (3, 512 * 1024 * 1024, 24, PoolKind.PROCESS, 3), + (400, 512 * 1024 * 1024, 6, PoolKind.PROCESS, 6), + ], +) +def test_resolve_archive_ingest_dispatch_tiers_on_measured_walk_size( monkeypatch: pytest.MonkeyPatch, + path_count: int, + total_bytes: int, + cpu_count: int, + expected_kind: PoolKind, + expected_workers: int, ) -> None: - monkeypatch.setattr("polylogue.pipeline.services.process_pool.available_cpus", lambda **_: 9) - monkeypatch.setattr(sys, "_is_gil_enabled", lambda: True, raising=False) - plan = resolve_archive_ingest_dispatch() - assert plan.pool_kind is PoolKind.PROCESS - # GIL build: min(8, cpus-1) = min(8, 8) = 8. - assert plan.worker_count == 8 + """The plan is sized from the work the walk found, not from CPU count alone. + + Anti-vacuity: dropping either byte tier, or the ``path_count`` term in the + ``min``, changes at least one row here. Before the tiers existed every row + resolved to PROCESS with the ambient ceiling, so a one-file walk spawned 16 + interpreters to parse one file. + """ + monkeypatch.setattr("polylogue.pipeline.services.process_pool.available_cpus", lambda **_: cpu_count) + plan = resolve_archive_ingest_dispatch(path_count=path_count, total_bytes=total_bytes, worker_ceiling=16) + assert plan.pool_kind is expected_kind + assert plan.worker_count == expected_workers -def test_resolve_archive_ingest_dispatch_honors_explicit_override() -> None: - """``parse_workers`` (the demo seeder's force-sequential knob) wins over - the ambient CPU-based default, clamped to at least 1.""" - plan = resolve_archive_ingest_dispatch(parse_workers=1) +def test_resolve_archive_ingest_dispatch_honors_worker_ceiling(monkeypatch: pytest.MonkeyPatch) -> None: + """The caller's resolved ``POLYLOGUE_INGEST_PARSE_WORKERS`` ceiling still binds.""" + monkeypatch.setattr("polylogue.pipeline.services.process_pool.available_cpus", lambda **_: 24) + plan = resolve_archive_ingest_dispatch(path_count=400, total_bytes=512 * 1024 * 1024, worker_ceiling=2) assert plan.pool_kind is PoolKind.PROCESS - assert plan.worker_count == 1 - - plan_negative = resolve_archive_ingest_dispatch(parse_workers=-5) - assert plan_negative.worker_count == 1 + assert plan.worker_count == 2 @pytest.mark.parametrize( From 68bef3b2b2528cc79b687ffd3edbc6035271622e Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 19:31:39 +0200 Subject: [PATCH 39/47] fix: Correct embedding preservation safety and scale findings Restore batches host parameters below the connection's own variable limit, writes a vector and its metadata together so a metadata row (the tier's reuse signal) can never exist without the vector at its address, and reports every unrestored hash as a typed miss naming its cause. Preservation builds the copy in a private temporary file, makes it self-contained, derives its receipt from the finished copy, and renames it into place, so a file at the destination path is always a whole copy its receipt describes. Deletion re-reads the copy's digest and refuses a receipt that names another file or a copy that has changed. Co-Authored-By: Claude Opus 5 --- .../maintenance/embedding_preservation.py | 260 +++++++++++--- .../test_embedding_preservation.py | 322 ++++++++++++++++-- 2 files changed, 506 insertions(+), 76 deletions(-) diff --git a/polylogue/maintenance/embedding_preservation.py b/polylogue/maintenance/embedding_preservation.py index 7fe803af32..9e76736aff 100644 --- a/polylogue/maintenance/embedding_preservation.py +++ b/polylogue/maintenance/embedding_preservation.py @@ -4,10 +4,17 @@ import hashlib import json +import os import sqlite3 +import tempfile +from collections.abc import Iterator, Sequence +from contextlib import closing from dataclasses import asdict, dataclass +from enum import StrEnum from pathlib import Path +from polylogue.core.durable_fs import sync_directory, write_once +from polylogue.storage.sqlite.archive_tiers.embeddings import EMBEDDING_DIMENSION from polylogue.storage.sqlite.sqlite_vec_extension import try_load_sqlite_vec _VECTOR_TABLES = ( @@ -20,6 +27,25 @@ ) _CURRENT_HASH_COLUMN = "vector_derivation_hash" _LEGACY_HASH_COLUMN = "embedding_input_hash" +_META_FIELDS = ("model", "dimension", "embedded_at_ms", "recipe_hash", "output_contract_hash") +# Hashes per IN list. Bounded by the connection's own variable limit, which is +# 999 on a default SQLite build and must never be assumed larger. +_MAX_HASH_BATCH = 500 + + +class RestoreMissReason(StrEnum): + """Why a wanted hash did not restore.""" + + METADATA_ABSENT = "metadata_absent" + METADATA_INCOMPLETE = "metadata_incomplete" + VECTOR_ABSENT = "vector_absent" + + +@dataclass(frozen=True, slots=True) +class RestoreMiss: + input_hash: str + reason: RestoreMissReason + detail: str = "" @dataclass(frozen=True, slots=True) @@ -30,7 +56,7 @@ class EmbeddingPreservationReceipt: vector_rows: int table_set_digest: str restored_hashes: int = 0 - missing_hashes: tuple[str, ...] = () + misses: tuple[RestoreMiss, ...] = () def _connect(path: Path, *, readonly: bool) -> sqlite3.Connection: @@ -56,8 +82,12 @@ def _table_digest(conn: sqlite3.Connection) -> tuple[str, dict[str, int]]: return digest.hexdigest(), counts +def _columns(conn: sqlite3.Connection, table: str) -> set[str]: + return {str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})")} + + def _hash_column(conn: sqlite3.Connection, table: str) -> str: - columns = {str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})")} + columns = _columns(conn, table) if _CURRENT_HASH_COLUMN in columns: return _CURRENT_HASH_COLUMN if _LEGACY_HASH_COLUMN in columns: @@ -65,19 +95,67 @@ def _hash_column(conn: sqlite3.Connection, table: str) -> str: raise RuntimeError(f"{table} has no supported embedding hash column") +def _hash_batches(conn: sqlite3.Connection, values: Sequence[bytes]) -> Iterator[Sequence[bytes]]: + """Chunk host parameters below this connection's own variable limit.""" + size = max(1, min(_MAX_HASH_BATCH, conn.getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER))) + for start in range(0, len(values), size): + yield values[start : start + size] + + +def _receipt_path(copy_path: Path) -> Path: + return copy_path.with_suffix(copy_path.suffix + ".receipt.json") + + +def _fsync_file(path: Path) -> None: + handle = os.open(path, os.O_RDONLY) + try: + os.fsync(handle) + finally: + os.close(handle) + + +def _fsync_directory(path: Path) -> None: + sync_directory(path) + + def preserve_embedding_vectors(source: str | Path, destination: str | Path) -> EmbeddingPreservationReceipt: - """Checkpoint-copy an embeddings database and record its vector population.""" + """Checkpoint-copy an embeddings database and record its vector population. + + The copy is built in a private temporary file and renamed into place only + once the backup has finished and the receipt has been derived from the + finished copy, so a file at the destination path is always a whole copy + that its receipt describes. + """ source_path = Path(source).absolute() destination_path = Path(destination).absolute() if source_path == destination_path: raise ValueError("embedding preservation source and copy must differ") destination_path.parent.mkdir(parents=True, exist_ok=True) - with _connect(source_path, readonly=True) as source_conn: - digest, counts = _table_digest(source_conn) - if destination_path.exists(): - raise FileExistsError(destination_path) - with sqlite3.connect(destination_path) as copy_conn: + if destination_path.exists(): + raise FileExistsError(destination_path) + handle, partial_name = tempfile.mkstemp( + dir=destination_path.parent, prefix=f".{destination_path.name}.", suffix=".partial" + ) + os.close(handle) + partial = Path(partial_name) + try: + with ( + closing(_connect(source_path, readonly=True)) as source_conn, + closing(sqlite3.connect(partial)) as copy_conn, + ): source_conn.backup(copy_conn) + # The copy inherits the source's journal mode, and a rename moves + # only the main file: the archived copy is made self-contained so + # it can never be separated from a WAL holding its content. + copy_conn.execute("PRAGMA journal_mode=DELETE").fetchall() + with closing(_connect(partial, readonly=True)) as copy_reader: + digest, counts = _table_digest(copy_reader) + _fsync_file(partial) + os.replace(partial, destination_path) + _fsync_directory(destination_path.parent) + except BaseException: + partial.unlink(missing_ok=True) + raise receipt = EmbeddingPreservationReceipt( source=str(source_path), copy=str(destination_path), @@ -85,12 +163,77 @@ def preserve_embedding_vectors(source: str | Path, destination: str | Path) -> E vector_rows=counts["message_embeddings"], table_set_digest=digest, ) - destination_path.with_suffix(destination_path.suffix + ".receipt.json").write_text( - json.dumps(asdict(receipt), indent=2, sort_keys=True) + "\n", encoding="utf-8" + write_once( + _receipt_path(destination_path), + (json.dumps(asdict(receipt), indent=2, sort_keys=True) + "\n").encode("utf-8"), ) return receipt +@dataclass(frozen=True, slots=True) +class _PreservedMetadata: + """A preserved row in the shape the current tier requires.""" + + model: str + dimension: int + embedded_at_ms: int | None + recipe_hash: bytes + output_contract_hash: bytes + + +def _validated_metadata(fields: dict[str, object]) -> _PreservedMetadata | str: + """The row as the current tier requires it, or the field that disqualifies it. + + ``message_embeddings_meta`` is complete by schema: a preserved row whose + model, dimension, or derivation identity is absent describes an output + nobody can vouch for, so it cannot stand in for a fresh embedding. + """ + model = fields.get("model") + if not isinstance(model, str) or not model: + return "model" + dimension = fields.get("dimension") + if not isinstance(dimension, int) or dimension != EMBEDDING_DIMENSION: + return "dimension" + identities: dict[str, bytes] = {} + for name in ("recipe_hash", "output_contract_hash"): + value = fields.get(name) + if not isinstance(value, (bytes, bytearray, memoryview)) or len(value) != 32: + return name + identities[name] = bytes(value) + embedded_at_ms = fields.get("embedded_at_ms") + return _PreservedMetadata( + model=model, + dimension=dimension, + embedded_at_ms=embedded_at_ms if isinstance(embedded_at_ms, int) else None, + recipe_hash=identities["recipe_hash"], + output_contract_hash=identities["output_contract_hash"], + ) + + +def _preserved_metadata( + conn: sqlite3.Connection, hash_column: str, projection: Sequence[str], batch: Sequence[bytes] +) -> dict[bytes, dict[str, object]]: + columns = ", ".join((hash_column, *projection)) + placeholders = ",".join("?" for _ in batch) + rows = conn.execute( + f"SELECT {columns} FROM message_embeddings_meta WHERE {hash_column} IN ({placeholders})", + tuple(batch), + ).fetchall() + return {bytes(row[0]): dict(zip(projection, row[1:], strict=True)) for row in rows} + + +def _preserved_vectors( + conn: sqlite3.Connection, hash_column: str, batch: Sequence[bytes] +) -> dict[bytes, tuple[object, object]]: + addresses = [value.hex() for value in batch] + placeholders = ",".join("?" for _ in addresses) + rows = conn.execute( + f"SELECT {hash_column}, embedding, model FROM message_embeddings WHERE {hash_column} IN ({placeholders})", + addresses, + ).fetchall() + return {bytes.fromhex(str(row[0])): (row[1], row[2]) for row in rows} + + def restore_embedding_vectors( destination: str | Path, preserved_copy: str | Path, @@ -98,64 +241,84 @@ def restore_embedding_vectors( ) -> EmbeddingPreservationReceipt: """Import preserved vectors for ``input_hashes`` into a fresh embeddings DB. - Metadata and vectors are write-once by input hash. Refs and lifecycle rows - remain owned by the fresh database and are created by normal convergence. + Metadata and vectors are write-once by input hash and are written together + in one transaction: a metadata row is the tier's reuse signal, so it may + never exist without the vector at its address. A hash counts as restored + only once both rows are present; every other outcome is an enumerated miss + carrying its cause. Refs and lifecycle rows remain owned by the fresh + database and are created by normal convergence. """ destination_path = Path(destination).absolute() copy_path = Path(preserved_copy).absolute() wanted = sorted(input_hashes) - with _connect(destination_path, readonly=False) as target, _connect(copy_path, readonly=True) as source: - source_meta_hash = _hash_column(source, "message_embeddings_meta") - source_vector_hash = _hash_column(source, "message_embeddings") - if wanted: - placeholders = ",".join("?" for _ in wanted) - rows = source.execute( - f"SELECT {source_meta_hash}, model, dimension, embedded_at_ms, recipe_hash, output_contract_hash " - f"FROM message_embeddings_meta WHERE {source_meta_hash} IN ({placeholders})", - wanted, - ).fetchall() - found = {bytes(row[0]) for row in rows} - for row in rows: + restored = 0 + misses: list[RestoreMiss] = [] + with ( + closing(_connect(destination_path, readonly=False)) as target, + closing(_connect(copy_path, readonly=True)) as source, + ): + meta_hash_column = _hash_column(source, "message_embeddings_meta") + vector_hash_column = _hash_column(source, "message_embeddings") + projection = [name for name in _META_FIELDS if name in _columns(source, "message_embeddings_meta")] + for batch in _hash_batches(source, wanted): + preserved = _preserved_metadata(source, meta_hash_column, projection, batch) + vectors = _preserved_vectors(source, vector_hash_column, batch) + for value in batch: + fields = preserved.get(value) + if fields is None: + misses.append(RestoreMiss(value.hex(), RestoreMissReason.METADATA_ABSENT)) + continue + record = _validated_metadata(fields) + if isinstance(record, str): + misses.append(RestoreMiss(value.hex(), RestoreMissReason.METADATA_INCOMPLETE, record)) + continue + vector = vectors.get(value) + if vector is None: + misses.append(RestoreMiss(value.hex(), RestoreMissReason.VECTOR_ABSENT)) + continue + target.execute( + "INSERT OR IGNORE INTO message_embeddings (vector_derivation_hash, embedding, model) " + "VALUES (?, ?, ?)", + (value.hex(), vector[0], vector[1]), + ) target.execute( "INSERT OR IGNORE INTO message_embeddings_meta " "(vector_derivation_hash, model, dimension, embedded_at_ms, recipe_hash, output_contract_hash) " "VALUES (?, ?, ?, ?, ?, ?)", - row, - ) - vector = source.execute( - f"SELECT embedding, model FROM message_embeddings WHERE {source_vector_hash} = ?", - (bytes(row[0]).hex(),), - ).fetchone() - if vector is None: - continue - target.execute( - "INSERT OR IGNORE INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", - (bytes(row[0]).hex(), vector[0], vector[1]), + ( + value, + record.model, + record.dimension, + record.embedded_at_ms, + record.recipe_hash, + record.output_contract_hash, + ), ) + restored += 1 target.commit() - else: - found = set() digest, counts = _table_digest(source) - missing = tuple(value.hex() for value in wanted if value not in found) return EmbeddingPreservationReceipt( source=str(copy_path), copy=str(destination_path), metadata_rows=counts["message_embeddings_meta"], vector_rows=counts["message_embeddings"], table_set_digest=digest, - restored_hashes=len(found), - missing_hashes=missing, + restored_hashes=restored, + misses=tuple(misses), ) def delete_preserved_copy(path: str | Path, *, receipt_path: str | Path | None = None) -> None: - """Delete a preservation copy only when an AC2 receipt authorizes it.""" + """Delete a preservation copy only when an AC2 receipt proves it is this copy. + + The receipt must name this file and carry the table-set digest the copy + still has, so a receipt filed for one copy can never authorize deleting + another, nor a copy that has changed since it was proven. + """ copy_path = Path(path).absolute() if not copy_path.is_file() or copy_path.is_symlink(): raise FileNotFoundError(copy_path) - if receipt_path is None: - receipt_path = copy_path.with_suffix(copy_path.suffix + ".receipt.json") - receipt = Path(receipt_path).absolute() + receipt = Path(receipt_path).absolute() if receipt_path is not None else _receipt_path(copy_path) if not receipt.is_file() or receipt.is_symlink(): raise FileNotFoundError(receipt) try: @@ -164,12 +327,21 @@ def delete_preserved_copy(path: str | Path, *, receipt_path: str | Path | None = raise ValueError("preservation deletion receipt is not valid JSON") from exc if proof.get("ac2_passed") is not True: raise ValueError("preservation copy requires an AC2-passed receipt before deletion") + named = proof.get("copy") + if not isinstance(named, str) or Path(named).absolute() != copy_path: + raise ValueError("preservation receipt names a different copy") + with closing(_connect(copy_path, readonly=True)) as conn: + digest, _counts = _table_digest(conn) + if digest != proof.get("table_set_digest"): + raise ValueError("preservation copy no longer matches its receipt digest") copy_path.unlink() receipt.unlink() __all__ = [ "EmbeddingPreservationReceipt", + "RestoreMiss", + "RestoreMissReason", "delete_preserved_copy", "preserve_embedding_vectors", "restore_embedding_vectors", diff --git a/tests/unit/maintenance/test_embedding_preservation.py b/tests/unit/maintenance/test_embedding_preservation.py index ee1f3a6f50..968eed2c85 100644 --- a/tests/unit/maintenance/test_embedding_preservation.py +++ b/tests/unit/maintenance/test_embedding_preservation.py @@ -2,11 +2,18 @@ import json import sqlite3 +import subprocess +import sys +from contextlib import closing from pathlib import Path +from typing import Any import pytest +from polylogue.maintenance import embedding_preservation from polylogue.maintenance.embedding_preservation import ( + RestoreMissReason, + _receipt_path, delete_preserved_copy, preserve_embedding_vectors, restore_embedding_vectors, @@ -18,30 +25,50 @@ _HASH = b"h" * 32 _OTHER = b"o" * 32 _MISSING = b"m" * 32 +_VECTOR = b"\x00" * (1024 * 4) +_RECIPE = b"a" * 32 +_CONTRACT = b"b" * 32 -def _db(path: Path, *, vector: bytes = _HASH) -> None: - initialize_archive_database(path, ArchiveTier.EMBEDDINGS) +def _open(path: Path) -> sqlite3.Connection: conn = sqlite3.connect(path) loaded, error = try_load_sqlite_vec(conn) if not loaded: conn.close() pytest.skip(str(error)) - conn.execute( - "INSERT INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", - (vector.hex(), b"\x00" * (1024 * 4), "test"), - ) - conn.execute( - "INSERT INTO message_embeddings_meta (vector_derivation_hash, model, dimension, recipe_hash, output_contract_hash) " - "VALUES (?, 'test', 1024, ?, ?)", - (vector, b"a" * 32, b"b" * 32), - ) - conn.commit() - conn.close() + return conn + + +def _db(path: Path, *, vectors: tuple[bytes, ...] = (_HASH,), metadata_only: tuple[bytes, ...] = ()) -> None: + """Current-schema embeddings DB holding ``vectors`` plus vector-less metadata rows.""" + initialize_archive_database(path, ArchiveTier.EMBEDDINGS) + with closing(_open(path)) as conn: + for value in (*vectors, *metadata_only): + if value in vectors: + conn.execute( + "INSERT INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", + (value.hex(), _VECTOR, "test"), + ) + conn.execute( + "INSERT INTO message_embeddings_meta " + "(vector_derivation_hash, model, dimension, recipe_hash, output_contract_hash) " + "VALUES (?, 'test', 1024, ?, ?)", + (value, _RECIPE, _CONTRACT), + ) + conn.commit() -def _legacy_db(path: Path, *, vector: bytes = _HASH) -> None: - with sqlite3.connect(path) as conn: +def _legacy_db( + path: Path, + *, + vectors: tuple[bytes, ...] = (_HASH,), + output_contract_hash: bytes | None = _CONTRACT, +) -> None: + """Pre-v5 embeddings DB: hashes named ``embedding_input_hash``, identity columns nullable. + + Mirrors the live archive DDL at /realm/state/polylogue/embeddings.db. + """ + with closing(sqlite3.connect(path)) as conn: conn.executescript( """ CREATE TABLE message_embeddings_meta ( @@ -59,14 +86,27 @@ def _legacy_db(path: Path, *, vector: bytes = _HASH) -> None: ); """ ) - conn.execute( - "INSERT INTO message_embeddings VALUES (?, ?, 'test')", - (vector.hex(), b"\x00" * (1024 * 4)), - ) - conn.execute( - "INSERT INTO message_embeddings_meta VALUES (?, 'test', 1024, NULL, ?, ?)", - (vector, b"a" * 32, b"b" * 32), - ) + for value in vectors: + conn.execute("INSERT INTO message_embeddings VALUES (?, ?, 'test')", (value.hex(), _VECTOR)) + conn.execute( + "INSERT INTO message_embeddings_meta VALUES (?, 'test', 1024, NULL, ?, ?)", + (value, _RECIPE, output_contract_hash), + ) + conn.commit() + + +def _count(path: Path, table: str) -> int: + with closing(_open(path)) as conn: + return int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + + +def _ac2_proof(receipt: Any, path: Path, **override: Any) -> Path: + """Write the preservation receipt back as an AC2-passed deletion proof.""" + from dataclasses import asdict + + proof = asdict(receipt) | {"ac2_passed": True} | override + path.write_text(json.dumps(proof), encoding="utf-8") + return path def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: @@ -74,7 +114,7 @@ def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: preserved = tmp_path / "preserved.db" fresh = tmp_path / "fresh.db" _db(source) - _db(fresh, vector=_OTHER) + _db(fresh, vectors=(_OTHER,)) before = preserve_embedding_vectors(source, preserved) assert before.metadata_rows == before.vector_rows == 1 @@ -82,8 +122,8 @@ def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: restored = restore_embedding_vectors(fresh, preserved, {_HASH}) assert restored.restored_hashes == 1 - assert restored.missing_hashes == () - with sqlite3.connect(fresh) as conn: + assert restored.misses == () + with closing(sqlite3.connect(fresh)) as conn: assert ( conn.execute( "SELECT COUNT(*) FROM message_embeddings_meta WHERE vector_derivation_hash = ?", (_HASH,) @@ -91,8 +131,7 @@ def test_preserve_restore_and_proof_deletion(tmp_path: Path) -> None: == 1 ) - proof = preserved.with_suffix(preserved.suffix + ".proof.json") - proof.write_text(json.dumps({"ac2_passed": True})) + proof = _ac2_proof(before, preserved.with_suffix(preserved.suffix + ".proof.json")) delete_preserved_copy(preserved, receipt_path=proof) assert not preserved.exists() assert not proof.exists() @@ -103,11 +142,13 @@ def test_missing_preserved_hash_is_enumerated(tmp_path: Path) -> None: preserved = tmp_path / "preserved.db" fresh = tmp_path / "fresh.db" _db(source) - _db(fresh, vector=_OTHER) + _db(fresh, vectors=(_OTHER,)) preserve_embedding_vectors(source, preserved) result = restore_embedding_vectors(fresh, preserved, {_HASH, _MISSING}) assert result.restored_hashes == 1 - assert result.missing_hashes == (_MISSING.hex(),) + assert [(miss.input_hash, miss.reason) for miss in result.misses] == [ + (_MISSING.hex(), RestoreMissReason.METADATA_ABSENT) + ] def test_restore_maps_legacy_embedding_input_hash_to_current_identity(tmp_path: Path) -> None: @@ -115,16 +156,233 @@ def test_restore_maps_legacy_embedding_input_hash_to_current_identity(tmp_path: preserved = tmp_path / "preserved.db" fresh = tmp_path / "fresh.db" _legacy_db(source) - _db(fresh, vector=_OTHER) + _db(fresh, vectors=(_OTHER,)) preserve_embedding_vectors(source, preserved) result = restore_embedding_vectors(fresh, preserved, {_HASH}) assert result.restored_hashes == 1 - with sqlite3.connect(fresh) as conn: + with closing(sqlite3.connect(fresh)) as conn: assert ( conn.execute( "SELECT COUNT(*) FROM message_embeddings_meta WHERE vector_derivation_hash = ?", (_HASH,) ).fetchone()[0] == 1 ) + + +def test_restore_batches_hashes_below_the_build_variable_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Red without batching: one IN list of six hashes exceeds a four-variable build.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + fresh = tmp_path / "fresh.db" + wanted = tuple(bytes([index]) * 32 for index in range(1, 7)) + _db(source, vectors=wanted) + _db(fresh, vectors=(_OTHER,)) + preserve_embedding_vectors(source, preserved) + + original = embedding_preservation._connect + + def small_limit(path: Any, *, readonly: bool) -> sqlite3.Connection: + conn = original(path, readonly=readonly) + if readonly: + conn.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 4) + return conn + + monkeypatch.setattr(embedding_preservation, "_connect", small_limit) + result = restore_embedding_vectors(fresh, preserved, set(wanted)) + + assert result.restored_hashes == len(wanted) + assert result.misses == () + + +def test_metadata_without_its_vector_is_a_miss_and_writes_nothing(tmp_path: Path) -> None: + """Red when metadata is written before its vector is found: a metadata row is the + tier's reuse signal, so one without a vector silently suppresses re-embedding.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + fresh = tmp_path / "fresh.db" + _db(source, vectors=(), metadata_only=(_HASH,)) + _db(fresh, vectors=(_OTHER,)) + preserve_embedding_vectors(source, preserved) + + result = restore_embedding_vectors(fresh, preserved, {_HASH}) + + assert result.restored_hashes == 0 + assert [(miss.input_hash, miss.reason) for miss in result.misses] == [ + (_HASH.hex(), RestoreMissReason.VECTOR_ABSENT) + ] + with closing(sqlite3.connect(fresh)) as conn: + assert ( + conn.execute( + "SELECT COUNT(*) FROM message_embeddings_meta WHERE vector_derivation_hash = ?", (_HASH,) + ).fetchone()[0] + == 0 + ) + + +def test_incomplete_legacy_metadata_is_a_typed_miss(tmp_path: Path) -> None: + """Red when an incomplete row is inserted: the current tier's NOT NULL identity + contract rejects it and aborts every remaining hash in the restore.""" + source = tmp_path / "legacy.db" + preserved = tmp_path / "preserved.db" + fresh = tmp_path / "fresh.db" + _legacy_db(source, vectors=(_HASH,), output_contract_hash=None) + _db(fresh, vectors=(_OTHER,)) + preserve_embedding_vectors(source, preserved) + + result = restore_embedding_vectors(fresh, preserved, {_HASH, _MISSING}) + + assert result.restored_hashes == 0 + assert [(miss.input_hash, miss.reason, miss.detail) for miss in result.misses] == [ + (_HASH.hex(), RestoreMissReason.METADATA_INCOMPLETE, "output_contract_hash"), + (_MISSING.hex(), RestoreMissReason.METADATA_ABSENT, ""), + ] + + +def test_deletion_refuses_a_receipt_naming_another_copy(tmp_path: Path) -> None: + """Red when the proof is not bound to the copy: any AC2-passed receipt authorizes + deleting any file.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + other = tmp_path / "other.db" + _db(source) + receipt = preserve_embedding_vectors(source, preserved) + other.write_bytes(preserved.read_bytes()) + + proof = _ac2_proof(receipt, tmp_path / "proof.json", copy=str(other)) + with pytest.raises(ValueError, match="different copy"): + delete_preserved_copy(preserved, receipt_path=proof) + assert preserved.exists() + + +def test_deletion_refuses_a_receipt_whose_digest_is_stale(tmp_path: Path) -> None: + """Red without a digest re-check: a copy mutated after its receipt still deletes.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + _db(source) + receipt = preserve_embedding_vectors(source, preserved) + with closing(_open(preserved)) as conn: + conn.execute("DELETE FROM message_embeddings WHERE vector_derivation_hash = ?", (_HASH.hex(),)) + conn.commit() + + proof = _ac2_proof(receipt, tmp_path / "proof.json") + with pytest.raises(ValueError, match="receipt digest"): + delete_preserved_copy(preserved, receipt_path=proof) + assert preserved.exists() + + +# Interrupting a backup from inside this process is impossible: sqlite3 discards +# exceptions raised in a progress callback, so the copy always runs to completion. +# The callback instead crashes the interpreter, which is the failure being guarded. +_CRASH_MID_BACKUP = """ +import os, sys +from pathlib import Path +from polylogue.maintenance import embedding_preservation as ep + +source, destination = Path(sys.argv[1]), Path(sys.argv[2]) +original = ep._connect + + +def interrupted(path, *, readonly): + conn = original(path, readonly=readonly) + if Path(path) != source: + return conn + + class Crash: + def __getattr__(self, name): + return getattr(conn, name) + + def __enter__(self): + conn.__enter__() + return self + + def __exit__(self, *exc): + return conn.__exit__(*exc) + + def backup(self, target, **kwargs): + conn.backup(target, pages=1, progress=lambda *_: os._exit(9)) + + return Crash() + + +ep._connect = interrupted +ep.preserve_embedding_vectors(source, destination) +""" + + +def test_a_crash_mid_backup_leaves_no_destination_file(tmp_path: Path) -> None: + """Red when the backup writes straight to the destination: the crash leaves a + truncated file there that no later run can tell apart from a whole copy.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + _db(source, vectors=tuple(bytes([index]) * 32 for index in range(1, 25))) + + crash = subprocess.run( + [sys.executable, "-c", _CRASH_MID_BACKUP, str(source), str(preserved)], + capture_output=True, + text=True, + ) + + assert crash.returncode == 9, crash.stderr + assert not preserved.exists() + assert not _receipt_path(preserved).exists() + + +def test_receipt_counts_come_from_the_completed_copy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Red when the receipt is read before the backup: a source written between the two + reads yields a receipt that describes no file.""" + source = tmp_path / "source.db" + preserved = tmp_path / "preserved.db" + _db(source) + + original = embedding_preservation._table_digest + grew = False + + def growing_source(conn: sqlite3.Connection) -> Any: + nonlocal grew + result = original(conn) + if not grew: + grew = True + with closing(_open(source)) as writer: + writer.execute( + "INSERT INTO message_embeddings (vector_derivation_hash, embedding, model) VALUES (?, ?, ?)", + (_OTHER.hex(), _VECTOR, "test"), + ) + writer.execute( + "INSERT INTO message_embeddings_meta " + "(vector_derivation_hash, model, dimension, recipe_hash, output_contract_hash) " + "VALUES (?, 'test', 1024, ?, ?)", + (_OTHER, _RECIPE, _CONTRACT), + ) + writer.commit() + return result + + monkeypatch.setattr(embedding_preservation, "_table_digest", growing_source) + receipt = preserve_embedding_vectors(source, preserved) + + assert receipt.metadata_rows == _count(preserved, "message_embeddings_meta") + assert receipt.vector_rows == _count(preserved, "message_embeddings") + with closing(embedding_preservation._connect(preserved, readonly=True)) as conn: + assert receipt.table_set_digest == original(conn)[0] + + +def test_preserved_copy_is_self_contained(tmp_path: Path) -> None: + """Red when the copy keeps the source's WAL mode: the rename moves only the main + file, leaving the copy beside sidecars that hold its content.""" + source = tmp_path / "source.db" + _db(source) + with closing(_open(source)) as conn: + conn.execute("PRAGMA journal_mode=WAL").fetchall() + vault = tmp_path / "vault" + preserved = vault / "preserved.db" + fresh = tmp_path / "fresh.db" + _db(fresh, vectors=(_OTHER,)) + + preserve_embedding_vectors(source, preserved) + + assert sorted(entry.name for entry in vault.iterdir()) == [ + "preserved.db", + "preserved.db.receipt.json", + ] + assert restore_embedding_vectors(fresh, preserved, {_HASH}).restored_hashes == 1 From bbaf1fb1c0e2b3e3007138b14859cbcfe94943ce Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 20:04:42 +0200 Subject: [PATCH 40/47] test: Cover fresh-archive attachment provenance round trip The envelope reader selects attachment_refs.direction and producer_ref unconditionally, so a schema missing either column fails every read at statement preparation. Assert the write-then-read round trip on a freshly bootstrapped index, and record the v91 declaration for the columns. Co-Authored-By: Claude Opus 5 --- .../storage/sqlite/archive_tiers/index.py | 3 ++ .../unit/storage/test_archive_tiers_write.py | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/polylogue/storage/sqlite/archive_tiers/index.py b/polylogue/storage/sqlite/archive_tiers/index.py index bc46f0d5a8..cbb7d60a0a 100644 --- a/polylogue/storage/sqlite/archive_tiers/index.py +++ b/polylogue/storage/sqlite/archive_tiers/index.py @@ -441,6 +441,9 @@ # their content hashes. # v90 adds canonical provider identity claims and parent-side dispatch # observations. Existing indexes must be replayed to reconstruct topology. +# v91 adds attachment reference provenance (`attachment_refs.direction` and +# `producer_ref`). Both are derived by the writer from the owning turn, so +# existing indexes must be replayed to recover them. # v92 removes content and cardinality pairing from delegation projection; # existing materialized rows must be regenerated from exact session links. # polylogue-vid0.1: v93 reads dispatch child identity from the progress diff --git a/tests/unit/storage/test_archive_tiers_write.py b/tests/unit/storage/test_archive_tiers_write.py index 5ea576daaa..9f1c878e54 100644 --- a/tests/unit/storage/test_archive_tiers_write.py +++ b/tests/unit/storage/test_archive_tiers_write.py @@ -5642,3 +5642,56 @@ def test_real_writer_persists_current_semantic_fingerprints_on_replay(tmp_path: assert tuple(replay) == (*tuple(first), "raw-replay") finally: conn.close() + + +def test_fresh_archive_reads_back_attachment_provenance_through_the_envelope(tmp_path: Path) -> None: + """A freshly bootstrapped index round-trips attachment direction and producer. + + The envelope reader selects ``attachment_refs.direction``/``producer_ref`` + unconditionally, so a fresh schema that omits either column fails at + statement preparation for every session, with or without attachments. + + Anti-vacuity: drop the ``direction`` and ``producer_ref`` columns from + ``ATTACHMENT_REFS_SPEC`` and this fails -- the write raises ``table + attachment_refs has no column named direction`` and the envelope read + raises ``no such column: r.direction``. + """ + conn = _connect(tmp_path / "index.db") + try: + session = ParsedSession( + source_name=Provider.CHATGPT, + provider_session_id="attachment-envelope-provenance", + messages=[ + ParsedMessage(provider_message_id="m1", role=Role.USER, text="here is my file"), + ParsedMessage(provider_message_id="m2", role=Role.ASSISTANT, text="here is the chart"), + ], + attachments=[ + ParsedAttachment( + provider_attachment_id="a1", + message_provider_id="m1", + name="input.txt", + mime_type="text/plain", + ), + ParsedAttachment( + provider_attachment_id="a2", + message_provider_id="m2", + name="chart.png", + mime_type="image/png", + ), + ], + ) + + session_id = write_parsed_session_to_archive(conn, session) + envelope = read_archive_session_envelope(conn, session_id) + + provenance = { + attachment.display_name: (attachment.direction, attachment.producer_ref) + for message in envelope.messages + for attachment in message.attachments + } + assert provenance["input.txt"] == ("user_input", None) + model_direction, model_producer = provenance["chart.png"] + assert model_direction == "model_output" + assert model_producer is not None + finally: + conn.close() From de09057e9114286d8bbaed077e9c7bb5512bbfae Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 16:43:25 +0200 Subject: [PATCH 41/47] test: add query execution envelope lab check --- devtools/command_catalog.py | 26 +++ devtools/query_execution_envelope.py | 202 ++++++++++++++++++ .../devtools/test_query_execution_envelope.py | 18 ++ 3 files changed, 246 insertions(+) create mode 100644 devtools/query_execution_envelope.py create mode 100644 tests/unit/devtools/test_query_execution_envelope.py diff --git a/devtools/command_catalog.py b/devtools/command_catalog.py index 4dfd43d3ff..0af57179b9 100644 --- a/devtools/command_catalog.py +++ b/devtools/command_catalog.py @@ -301,6 +301,32 @@ def to_dict(self) -> dict[str, object]: use_when="Assert memory budgets around a concrete query or archive-facing command.", examples=("devtools bench memory --max-rss-mb 1536 -- polylogue --plain analyze",), ), + CommandSpec( + "bench query-envelope", + "benchmarking", + "Measure repeated incident-scale query RSS, PSS, swap, and temp envelopes.", + "devtools.query_execution_envelope", + json_flag=False, + use_when="Run the opt-in live archive proof for repeated aggregate query_units calls and emit a receipt.", + examples=( + "devtools bench query-envelope --archive-root /path/to/archive --receipt .cache/query-envelope.json", + ), + ), + CommandSpec( + "archive index-fast-forward", + "archive", + "Plan and prove a declared index fast-forward against retained raw replay.", + "devtools.index_fast_forward", + use_when=( + "Advance a stopped index generation across a declared clone-safe schema gap. The actuator clones the " + "active generation, applies lifecycle operations, proves a deterministic retained-raw sample through " + "the production parser/materializer route, then atomically activates the proven generation." + ), + examples=( + "devtools archive index-fast-forward prepare --archive-root /path/to/archive --receipt /path/to/receipt.json", + "devtools archive index-fast-forward activate --receipt /path/to/receipt.json", + ), + ), CommandSpec( "archive lineage-validation", "archive", diff --git a/devtools/query_execution_envelope.py b/devtools/query_execution_envelope.py new file mode 100644 index 0000000000..fea2dbb543 --- /dev/null +++ b/devtools/query_execution_envelope.py @@ -0,0 +1,202 @@ +"""Measure repeated incident-scale query resource envelopes. + +The command is intentionally an opt-in lab check. It opens the supplied +archive through the public read route and never writes to the archive. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import threading +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from shutil import disk_usage +from typing import Any + +from polylogue import Polylogue + +DEFAULT_EXPRESSION = "actions where tool:shell | group by tool, session.origin | count" +DEFAULT_ROUNDS = 20 +DEFAULT_WARMUP = 3 +DEFAULT_BASELINE = 5 +DEFAULT_TOLERANCE = 0.25 + + +@dataclass(frozen=True, slots=True) +class ResourceSample: + rss_bytes: int + pss_bytes: int + swap_bytes: int + temp_delta_bytes: int + + +def _proc_memory() -> tuple[int, int, int]: + """Return current RSS, PSS, and swap from procfs for this process.""" + rss = pss = swap = 0 + try: + for line in (Path("/proc/self/status").read_text() + Path("/proc/self/smaps_rollup").read_text()).splitlines(): + key, _, value = line.partition(":") + fields = value.split() + if not fields: + continue + if key == "VmRSS": + rss = int(fields[0]) * 1024 + elif key == "Pss": + pss = int(fields[0]) * 1024 + elif key == "VmSwap": + swap = int(fields[0]) * 1024 + except OSError: + pass + return rss, pss, swap + + +def _temp_used_bytes(temp_root: Path) -> int: + try: + usage = disk_usage(temp_root) + except OSError: + return 0 + return usage.total - usage.free + + +async def _query_once(archive: Polylogue, expression: str) -> dict[str, Any]: + envelope = await archive.query_units(expression, limit=100) + return envelope.model_dump(mode="json") + + +async def measure_query_envelope( + archive_root: Path, + *, + expression: str = DEFAULT_EXPRESSION, + rounds: int = DEFAULT_ROUNDS, + warmup: int = DEFAULT_WARMUP, + baseline_rounds: int = DEFAULT_BASELINE, + tolerance: float = DEFAULT_TOLERANCE, + sample_interval_s: float = 0.05, +) -> dict[str, Any]: + """Run repeated aggregate reads and return a bounded resource receipt.""" + if rounds < 20: + raise ValueError("rounds must be at least 20") + if warmup < 0 or baseline_rounds < 1: + raise ValueError("warmup must be non-negative and baseline_rounds must be positive") + archive_root = archive_root.resolve() + db_path = archive_root / "index.db" + if not db_path.is_file(): + raise FileNotFoundError(db_path) + temp_root = Path(os.environ.get("TMPDIR", "/tmp")) + temp_before = _temp_used_bytes(temp_root) + peak = ResourceSample(0, 0, 0, 0) + stop = threading.Event() + + def sample() -> None: + nonlocal peak + while not stop.is_set(): + rss, pss, swap = _proc_memory() + candidate = ResourceSample(rss, pss, swap, max(0, _temp_used_bytes(temp_root) - temp_before)) + peak = ResourceSample( + max(peak.rss_bytes, candidate.rss_bytes), + max(peak.pss_bytes, candidate.pss_bytes), + max(peak.swap_bytes, candidate.swap_bytes), + max(peak.temp_delta_bytes, candidate.temp_delta_bytes), + ) + time.sleep(sample_interval_s) + + sampler = threading.Thread(target=sample, name="query-envelope-sampler", daemon=True) + sampler.start() + started = time.perf_counter() + result_counts: list[int] = [] + per_round: list[ResourceSample] = [] + try: + async with Polylogue(archive_root=archive_root, db_path=db_path) as archive: + for _ in range(warmup): + result_counts.append(len((await _query_once(archive, expression)).get("items", []))) + for _ in range(baseline_rounds + rounds): + result_counts.append(len((await _query_once(archive, expression)).get("items", []))) + rss, pss, swap = _proc_memory() + per_round.append(ResourceSample(rss, pss, swap, max(0, _temp_used_bytes(temp_root) - temp_before))) + finally: + stop.set() + sampler.join(timeout=2) + elapsed_ms = round((time.perf_counter() - started) * 1000, 3) + baseline = per_round[:baseline_rounds] + measured = per_round[baseline_rounds:] + baseline_rss = max(sample.rss_bytes for sample in baseline) + baseline_pss = max(sample.pss_bytes for sample in baseline) + baseline_swap = max(sample.swap_bytes for sample in baseline) + baseline_temp = max(sample.temp_delta_bytes for sample in baseline) + final = measured[-3:] + returned = ( + all( + current <= max(1, baseline_value) * (1 + tolerance) + for current, baseline_value in ((sample.rss_bytes, baseline_rss) for sample in final) + ) + and all( + current <= max(1, baseline_value) * (1 + tolerance) + for current, baseline_value in ((sample.pss_bytes, baseline_pss) for sample in final) + ) + and all(sample.swap_bytes <= baseline_swap + 64 * 1024 * 1024 for sample in final) + ) + receipt = { + "status": "succeeded" if returned else "failed", + "archive_root": str(archive_root), + "archive_index_bytes": db_path.stat().st_size, + "expression": expression, + "rounds": rounds, + "warmup_rounds": warmup, + "baseline_rounds": baseline_rounds, + "tolerance": tolerance, + "steady_state_envelope": { + "rss_bytes": baseline_rss, + "pss_bytes": baseline_pss, + "swap_bytes": baseline_swap, + "temp_delta_bytes": baseline_temp, + "return_tolerance": tolerance, + }, + "peak": asdict(peak), + "final_samples": [asdict(sample) for sample in final], + "result_item_counts": result_counts, + "elapsed_ms": elapsed_ms, + "returned_to_envelope": returned, + "regression_path": "rerun this command against the same promoted generation; status=failed identifies envelope drift", + } + return receipt + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archive-root", type=Path, required=True) + parser.add_argument("--rounds", type=int, default=DEFAULT_ROUNDS) + parser.add_argument("--warmup", type=int, default=DEFAULT_WARMUP) + parser.add_argument("--baseline-rounds", type=int, default=DEFAULT_BASELINE) + parser.add_argument("--tolerance", type=float, default=DEFAULT_TOLERANCE) + parser.add_argument("--sample-interval", type=float, default=0.05) + parser.add_argument("--expression", default=DEFAULT_EXPRESSION) + parser.add_argument("--receipt", type=Path) + args = parser.parse_args(argv) + try: + receipt = asyncio.run( + measure_query_envelope( + args.archive_root, + expression=args.expression, + rounds=args.rounds, + warmup=args.warmup, + baseline_rounds=args.baseline_rounds, + tolerance=args.tolerance, + sample_interval_s=args.sample_interval, + ) + ) + except (FileNotFoundError, ValueError) as exc: + parser.error(str(exc)) + text = json.dumps(receipt, indent=2, sort_keys=True) + print(text) + if args.receipt: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(text + "\n", encoding="utf-8") + return 0 if receipt["returned_to_envelope"] else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/devtools/test_query_execution_envelope.py b/tests/unit/devtools/test_query_execution_envelope.py new file mode 100644 index 0000000000..4e2ba1ca1f --- /dev/null +++ b/tests/unit/devtools/test_query_execution_envelope.py @@ -0,0 +1,18 @@ +"""Tests for the live query execution envelope lab command.""" + +from __future__ import annotations + +from pathlib import Path + +from devtools.query_execution_envelope import _proc_memory, _temp_used_bytes + + +def test_proc_memory_is_nonnegative() -> None: + rss, pss, swap = _proc_memory() + assert rss >= 0 + assert pss >= 0 + assert swap >= 0 + + +def test_temp_usage_missing_path_is_zero(tmp_path: Path) -> None: + assert _temp_used_bytes(tmp_path / "missing") == 0 From d53f3259a35e348d127cc1861cb4bb4dc3e98ce1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 16:59:32 +0200 Subject: [PATCH 42/47] feat: add query execution envelope lab check --- devtools/query_execution_envelope.py | 149 +++++++++++++----- docs/devtools.md | 2 + .../devtools/test_query_execution_envelope.py | 102 +++++++++++- 3 files changed, 211 insertions(+), 42 deletions(-) diff --git a/devtools/query_execution_envelope.py b/devtools/query_execution_envelope.py index fea2dbb543..cf52d0d12d 100644 --- a/devtools/query_execution_envelope.py +++ b/devtools/query_execution_envelope.py @@ -1,7 +1,7 @@ """Measure repeated incident-scale query resource envelopes. -The command is intentionally an opt-in lab check. It opens the supplied -archive through the public read route and never writes to the archive. +The command is an opt-in lab check. It opens the supplied archive through the +public query route and never writes to the archive. """ from __future__ import annotations @@ -19,15 +19,22 @@ from polylogue import Polylogue -DEFAULT_EXPRESSION = "actions where tool:shell | group by tool, session.origin | count" +DEFAULT_EXPRESSION = "actions where tool:shell | group by tool | count" DEFAULT_ROUNDS = 20 DEFAULT_WARMUP = 3 DEFAULT_BASELINE = 5 DEFAULT_TOLERANCE = 0.25 +DEFAULT_MAX_RSS_MB = 1536 +DEFAULT_MAX_PSS_MB = 1536 +DEFAULT_MAX_SWAP_GROWTH_MB = 64 +DEFAULT_MAX_TEMP_GROWTH_MB = 64 +MIB = 1024 * 1024 @dataclass(frozen=True, slots=True) class ResourceSample: + """One process and temporary-filesystem observation.""" + rss_bytes: int pss_bytes: int swap_bytes: int @@ -38,7 +45,9 @@ def _proc_memory() -> tuple[int, int, int]: """Return current RSS, PSS, and swap from procfs for this process.""" rss = pss = swap = 0 try: - for line in (Path("/proc/self/status").read_text() + Path("/proc/self/smaps_rollup").read_text()).splitlines(): + status = Path("/proc/self/status").read_text(encoding="ascii") + smaps_rollup = Path("/proc/self/smaps_rollup").read_text(encoding="ascii") + for line in (status + smaps_rollup).splitlines(): key, _, value = line.partition(":") fields = value.split() if not fields: @@ -49,12 +58,13 @@ def _proc_memory() -> tuple[int, int, int]: pss = int(fields[0]) * 1024 elif key == "VmSwap": swap = int(fields[0]) * 1024 - except OSError: + except (OSError, ValueError): pass return rss, pss, swap def _temp_used_bytes(temp_root: Path) -> int: + """Return used bytes on the filesystem containing the temp root.""" try: usage = disk_usage(temp_root) except OSError: @@ -76,93 +86,142 @@ async def measure_query_envelope( baseline_rounds: int = DEFAULT_BASELINE, tolerance: float = DEFAULT_TOLERANCE, sample_interval_s: float = 0.05, + max_rss_bytes: int = DEFAULT_MAX_RSS_MB * MIB, + max_pss_bytes: int = DEFAULT_MAX_PSS_MB * MIB, + max_swap_growth_bytes: int = DEFAULT_MAX_SWAP_GROWTH_MB * MIB, + max_temp_growth_bytes: int = DEFAULT_MAX_TEMP_GROWTH_MB * MIB, ) -> dict[str, Any]: - """Run repeated aggregate reads and return a bounded resource receipt.""" + """Run repeated aggregate reads and return a resource receipt.""" if rounds < 20: raise ValueError("rounds must be at least 20") if warmup < 0 or baseline_rounds < 1: raise ValueError("warmup must be non-negative and baseline_rounds must be positive") + if tolerance < 0 or sample_interval_s < 0: + raise ValueError("tolerance and sample interval must be non-negative") + if min(max_rss_bytes, max_pss_bytes, max_swap_growth_bytes, max_temp_growth_bytes) < 0: + raise ValueError("resource envelope limits must be non-negative") + archive_root = archive_root.resolve() db_path = archive_root / "index.db" if not db_path.is_file(): raise FileNotFoundError(db_path) + temp_root = Path(os.environ.get("TMPDIR", "/tmp")) temp_before = _temp_used_bytes(temp_root) - peak = ResourceSample(0, 0, 0, 0) + initial_rss, initial_pss, initial_swap = _proc_memory() + initial = ResourceSample(initial_rss, initial_pss, initial_swap, 0) + peak = initial stop = threading.Event() - def sample() -> None: + def observe() -> ResourceSample: nonlocal peak + rss, pss, swap = _proc_memory() + candidate = ResourceSample(rss, pss, swap, max(0, _temp_used_bytes(temp_root) - temp_before)) + peak = ResourceSample( + max(peak.rss_bytes, candidate.rss_bytes), + max(peak.pss_bytes, candidate.pss_bytes), + max(peak.swap_bytes, candidate.swap_bytes), + max(peak.temp_delta_bytes, candidate.temp_delta_bytes), + ) + return candidate + + def sample() -> None: while not stop.is_set(): - rss, pss, swap = _proc_memory() - candidate = ResourceSample(rss, pss, swap, max(0, _temp_used_bytes(temp_root) - temp_before)) - peak = ResourceSample( - max(peak.rss_bytes, candidate.rss_bytes), - max(peak.pss_bytes, candidate.pss_bytes), - max(peak.swap_bytes, candidate.swap_bytes), - max(peak.temp_delta_bytes, candidate.temp_delta_bytes), - ) + observe() time.sleep(sample_interval_s) sampler = threading.Thread(target=sample, name="query-envelope-sampler", daemon=True) sampler.start() started = time.perf_counter() result_counts: list[int] = [] - per_round: list[ResourceSample] = [] + samples: list[dict[str, Any]] = [] try: async with Polylogue(archive_root=archive_root, db_path=db_path) as archive: - for _ in range(warmup): - result_counts.append(len((await _query_once(archive, expression)).get("items", []))) - for _ in range(baseline_rounds + rounds): - result_counts.append(len((await _query_once(archive, expression)).get("items", []))) - rss, pss, swap = _proc_memory() - per_round.append(ResourceSample(rss, pss, swap, max(0, _temp_used_bytes(temp_root) - temp_before))) + for phase, count in (("warmup", warmup), ("baseline", baseline_rounds), ("measured", rounds)): + for round_number in range(count): + result_count = len((await _query_once(archive, expression)).get("items", [])) + result_counts.append(result_count) + sample_now = observe() + samples.append( + { + "phase": phase, + "round": round_number + 1, + "result_item_count": result_count, + **asdict(sample_now), + } + ) + quiescent = observe() finally: stop.set() sampler.join(timeout=2) + elapsed_ms = round((time.perf_counter() - started) * 1000, 3) - baseline = per_round[:baseline_rounds] - measured = per_round[baseline_rounds:] + baseline = [ + ResourceSample(item["rss_bytes"], item["pss_bytes"], item["swap_bytes"], item["temp_delta_bytes"]) + for item in samples + if item["phase"] == "baseline" + ] + measured = [ + ResourceSample(item["rss_bytes"], item["pss_bytes"], item["swap_bytes"], item["temp_delta_bytes"]) + for item in samples + if item["phase"] == "measured" + ] baseline_rss = max(sample.rss_bytes for sample in baseline) baseline_pss = max(sample.pss_bytes for sample in baseline) baseline_swap = max(sample.swap_bytes for sample in baseline) baseline_temp = max(sample.temp_delta_bytes for sample in baseline) final = measured[-3:] - returned = ( - all( - current <= max(1, baseline_value) * (1 + tolerance) - for current, baseline_value in ((sample.rss_bytes, baseline_rss) for sample in final) - ) - and all( - current <= max(1, baseline_value) * (1 + tolerance) - for current, baseline_value in ((sample.pss_bytes, baseline_pss) for sample in final) - ) - and all(sample.swap_bytes <= baseline_swap + 64 * 1024 * 1024 for sample in final) - ) - receipt = { + return_checks = { + "rss": all(current.rss_bytes <= max(1, baseline_rss) * (1 + tolerance) for current in final), + "pss": all(current.pss_bytes <= max(1, baseline_pss) * (1 + tolerance) for current in final), + "swap": all(current.swap_bytes <= baseline_swap + max_swap_growth_bytes for current in final), + "temp": all(current.temp_delta_bytes <= baseline_temp + max_temp_growth_bytes for current in final), + } + absolute_checks = { + "rss": peak.rss_bytes <= max_rss_bytes, + "pss": peak.pss_bytes <= max_pss_bytes, + "swap": peak.swap_bytes <= initial_swap + max_swap_growth_bytes, + "temp": peak.temp_delta_bytes <= max_temp_growth_bytes, + } + returned = all(return_checks.values()) and all(absolute_checks.values()) + return { "status": "succeeded" if returned else "failed", "archive_root": str(archive_root), + "archive_generation": db_path.resolve().parent.name, "archive_index_bytes": db_path.stat().st_size, "expression": expression, "rounds": rounds, "warmup_rounds": warmup, "baseline_rounds": baseline_rounds, "tolerance": tolerance, - "steady_state_envelope": { + "declared_envelope": { + "max_rss_bytes": max_rss_bytes, + "max_pss_bytes": max_pss_bytes, + "max_swap_growth_bytes": max_swap_growth_bytes, + "max_temp_growth_bytes": max_temp_growth_bytes, + "return_tolerance": tolerance, + }, + "steady_state_baseline": { "rss_bytes": baseline_rss, "pss_bytes": baseline_pss, "swap_bytes": baseline_swap, "temp_delta_bytes": baseline_temp, - "return_tolerance": tolerance, }, "peak": asdict(peak), + "initial_sample": asdict(initial), + "quiescent_sample": asdict(quiescent), "final_samples": [asdict(sample) for sample in final], + "samples": samples, + "return_checks": return_checks, + "absolute_checks": absolute_checks, "result_item_counts": result_counts, "elapsed_ms": elapsed_ms, "returned_to_envelope": returned, - "regression_path": "rerun this command against the same promoted generation; status=failed identifies envelope drift", + "regression_path": ( + "Rerun this command against the same promoted generation. A failed status identifies RSS/PSS, " + "swap, temp, or return-to-baseline drift; compare the named check and samples." + ), } - return receipt def main(argv: list[str] | None = None) -> int: @@ -173,6 +232,10 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--baseline-rounds", type=int, default=DEFAULT_BASELINE) parser.add_argument("--tolerance", type=float, default=DEFAULT_TOLERANCE) parser.add_argument("--sample-interval", type=float, default=0.05) + parser.add_argument("--max-rss-mb", type=int, default=DEFAULT_MAX_RSS_MB) + parser.add_argument("--max-pss-mb", type=int, default=DEFAULT_MAX_PSS_MB) + parser.add_argument("--max-swap-growth-mb", type=int, default=DEFAULT_MAX_SWAP_GROWTH_MB) + parser.add_argument("--max-temp-growth-mb", type=int, default=DEFAULT_MAX_TEMP_GROWTH_MB) parser.add_argument("--expression", default=DEFAULT_EXPRESSION) parser.add_argument("--receipt", type=Path) args = parser.parse_args(argv) @@ -186,6 +249,10 @@ def main(argv: list[str] | None = None) -> int: baseline_rounds=args.baseline_rounds, tolerance=args.tolerance, sample_interval_s=args.sample_interval, + max_rss_bytes=args.max_rss_mb * MIB, + max_pss_bytes=args.max_pss_mb * MIB, + max_swap_growth_bytes=args.max_swap_growth_mb * MIB, + max_temp_growth_bytes=args.max_temp_growth_mb * MIB, ) ) except (FileNotFoundError, ValueError) as exc: diff --git a/docs/devtools.md b/docs/devtools.md index 53c9fa4953..d03fa81156 100644 --- a/docs/devtools.md +++ b/docs/devtools.md @@ -98,6 +98,7 @@ These are the commands worth remembering during normal repo work: | --- | --- | | `devtools bench memory` | Measure query-memory envelopes on generated fixtures. | | `devtools bench pipeline` | Run typed pipeline probes against synthetic, staged, or archive-subset inputs. | +| `devtools bench query-envelope` | Measure repeated incident-scale query RSS, PSS, swap, and temp envelopes. | | `devtools bench slo` | Check read-surface latency budgets in docs/plans/slo-catalog.yaml against benchmark measurements. | ### Archive @@ -105,6 +106,7 @@ These are the commands worth remembering during normal repo work: | Command | Description | | --- | --- | | `devtools archive continuity-evidence` | Replay continuity scenarios and verify their query routes are discoverable. | +| `devtools archive index-fast-forward` | Plan and prove a declared index fast-forward against retained raw replay. | | `devtools archive lineage-validation` | Validate lineage-count evidence before citing archive counts externally. | diff --git a/tests/unit/devtools/test_query_execution_envelope.py b/tests/unit/devtools/test_query_execution_envelope.py index 4e2ba1ca1f..e1b3bdd1a7 100644 --- a/tests/unit/devtools/test_query_execution_envelope.py +++ b/tests/unit/devtools/test_query_execution_envelope.py @@ -4,7 +4,10 @@ from pathlib import Path -from devtools.query_execution_envelope import _proc_memory, _temp_used_bytes +import pytest + +import devtools.query_execution_envelope as envelope_module +from devtools.query_execution_envelope import _proc_memory, _temp_used_bytes, measure_query_envelope def test_proc_memory_is_nonnegative() -> None: @@ -16,3 +19,100 @@ def test_proc_memory_is_nonnegative() -> None: def test_temp_usage_missing_path_is_zero(tmp_path: Path) -> None: assert _temp_used_bytes(tmp_path / "missing") == 0 + + +async def test_measure_query_envelope_runs_the_declared_repetition_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The receipt covers every query round and all four resource dimensions.""" + + (tmp_path / "index.db").write_bytes(b"synthetic index") + calls = 0 + + class FakeEnvelope: + def model_dump(self, *, mode: str) -> dict[str, object]: + assert mode == "json" + return {"items": [{"group_key": "shell", "count": 1}]} + + class FakePolylogue: + def __init__(self, **_kwargs: object) -> None: + pass + + async def __aenter__(self) -> FakePolylogue: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def query_units(self, expression: str, *, limit: int) -> FakeEnvelope: + nonlocal calls + assert expression == "actions where tool:shell | group by tool | count" + assert limit == 100 + calls += 1 + return FakeEnvelope() + + monkeypatch.setattr(envelope_module, "Polylogue", FakePolylogue) + monkeypatch.setattr(envelope_module, "_proc_memory", lambda: (100, 80, 0)) + monkeypatch.setattr(envelope_module, "_temp_used_bytes", lambda _root: 100) + + receipt = await measure_query_envelope( + tmp_path, + warmup=0, + baseline_rounds=2, + sample_interval_s=0.001, + max_rss_bytes=100, + max_pss_bytes=80, + max_swap_growth_bytes=0, + max_temp_growth_bytes=0, + ) + + assert calls == 22 + assert receipt["status"] == "succeeded" + assert len(receipt["samples"]) == 22 + assert len(receipt["final_samples"]) == 3 + assert receipt["return_checks"] == {"rss": True, "pss": True, "swap": True, "temp": True} + assert receipt["absolute_checks"] == {"rss": True, "pss": True, "swap": True, "temp": True} + + +async def test_measure_query_envelope_fails_when_declared_rss_is_exceeded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The absolute RSS declaration is a real failure condition.""" + + (tmp_path / "index.db").write_bytes(b"synthetic index") + + class FakeEnvelope: + def model_dump(self, *, mode: str) -> dict[str, object]: + return {"items": []} + + class FakePolylogue: + def __init__(self, **_kwargs: object) -> None: + pass + + async def __aenter__(self) -> FakePolylogue: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def query_units(self, _expression: str, *, limit: int) -> FakeEnvelope: + assert limit == 100 + return FakeEnvelope() + + monkeypatch.setattr(envelope_module, "Polylogue", FakePolylogue) + monkeypatch.setattr(envelope_module, "_proc_memory", lambda: (100, 80, 0)) + monkeypatch.setattr(envelope_module, "_temp_used_bytes", lambda _root: 100) + + receipt = await measure_query_envelope( + tmp_path, + warmup=0, + baseline_rounds=1, + sample_interval_s=0.001, + max_rss_bytes=99, + max_pss_bytes=80, + max_swap_growth_bytes=0, + max_temp_growth_bytes=0, + ) + + assert receipt["status"] == "failed" + assert receipt["absolute_checks"]["rss"] is False From 15d3d1a5a9c756b28aae1e8f685715787c9709db Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 17:02:01 +0200 Subject: [PATCH 43/47] fix: record incompatible envelope environments --- devtools/query_execution_envelope.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/devtools/query_execution_envelope.py b/devtools/query_execution_envelope.py index cf52d0d12d..09198a77f6 100644 --- a/devtools/query_execution_envelope.py +++ b/devtools/query_execution_envelope.py @@ -18,6 +18,7 @@ from typing import Any from polylogue import Polylogue +from polylogue.core.errors import SchemaVersionMismatchError DEFAULT_EXPRESSION = "actions where tool:shell | group by tool | count" DEFAULT_ROUNDS = 20 @@ -255,6 +256,22 @@ def main(argv: list[str] | None = None) -> int: max_temp_growth_bytes=args.max_temp_growth_mb * MIB, ) ) + except SchemaVersionMismatchError as exc: + receipt = { + "status": "blocked-env", + "archive_root": str(args.archive_root.resolve()), + "blocking_error": str(exc), + "regression_path": ( + "Re-run against the exact promoted generation after its schema lifecycle action completes; " + "do not bypass the archive compatibility check." + ), + } + text = json.dumps(receipt, indent=2, sort_keys=True) + print(text) + if args.receipt: + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(text + "\n", encoding="utf-8") + return 2 except (FileNotFoundError, ValueError) as exc: parser.error(str(exc)) text = json.dumps(receipt, indent=2, sort_keys=True) From 3db65f519d57395af0afc1a4968eea50cec363e3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:24:50 +0200 Subject: [PATCH 44/47] fix: drop resurrected index fast-forward command spec The rebase conflict resolution re-added the archive index-fast-forward CommandSpec that #4666 deleted along with devtools/index_fast_forward.py. Its module target does not exist. --- devtools/command_catalog.py | 15 --------------- docs/devtools.md | 1 - 2 files changed, 16 deletions(-) diff --git a/devtools/command_catalog.py b/devtools/command_catalog.py index 0af57179b9..c2092d38a5 100644 --- a/devtools/command_catalog.py +++ b/devtools/command_catalog.py @@ -312,21 +312,6 @@ def to_dict(self) -> dict[str, object]: "devtools bench query-envelope --archive-root /path/to/archive --receipt .cache/query-envelope.json", ), ), - CommandSpec( - "archive index-fast-forward", - "archive", - "Plan and prove a declared index fast-forward against retained raw replay.", - "devtools.index_fast_forward", - use_when=( - "Advance a stopped index generation across a declared clone-safe schema gap. The actuator clones the " - "active generation, applies lifecycle operations, proves a deterministic retained-raw sample through " - "the production parser/materializer route, then atomically activates the proven generation." - ), - examples=( - "devtools archive index-fast-forward prepare --archive-root /path/to/archive --receipt /path/to/receipt.json", - "devtools archive index-fast-forward activate --receipt /path/to/receipt.json", - ), - ), CommandSpec( "archive lineage-validation", "archive", diff --git a/docs/devtools.md b/docs/devtools.md index d03fa81156..7d90795afb 100644 --- a/docs/devtools.md +++ b/docs/devtools.md @@ -106,7 +106,6 @@ These are the commands worth remembering during normal repo work: | Command | Description | | --- | --- | | `devtools archive continuity-evidence` | Replay continuity scenarios and verify their query routes are discoverable. | -| `devtools archive index-fast-forward` | Plan and prove a declared index fast-forward against retained raw replay. | | `devtools archive lineage-validation` | Validate lineage-count evidence before citing archive counts externally. | From 11f9f1da90b15b5156d604a9775987dc41d44a89 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 4 Sep 2026 18:37:49 +0200 Subject: [PATCH 45/47] fix: refuse unmeasurable envelope samples A procfs read failure returned zero RSS, PSS and swap, which satisfies every declared limit and reports the envelope as met without measuring it. Missing fields now raise and the command records blocked-env. --- devtools/query_execution_envelope.py | 67 +++++++++++++------ .../devtools/test_query_execution_envelope.py | 24 ++++++- 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/devtools/query_execution_envelope.py b/devtools/query_execution_envelope.py index 09198a77f6..ceb88d8a23 100644 --- a/devtools/query_execution_envelope.py +++ b/devtools/query_execution_envelope.py @@ -42,26 +42,43 @@ class ResourceSample: temp_delta_bytes: int +PROC_MEMORY_FIELDS = ("VmRSS", "Pss", "VmSwap") + + +class ResourceProbeUnavailableError(RuntimeError): + """procfs did not report a field the declared envelope is measured against.""" + + +def _parse_proc_memory(text: str) -> tuple[int, int, int]: + """Return RSS, PSS, and swap in bytes from concatenated procfs reports. + + A missing field is refused rather than defaulted to zero: a zero sample + satisfies every declared limit, so the envelope would pass without + measuring anything. + """ + values: dict[str, int] = {} + for line in text.splitlines(): + key, _, value = line.partition(":") + if key not in PROC_MEMORY_FIELDS: + continue + fields = value.split() + if not fields: + continue + values[key] = int(fields[0]) * 1024 + missing = [field for field in PROC_MEMORY_FIELDS if field not in values] + if missing: + raise ResourceProbeUnavailableError(f"procfs did not report {', '.join(missing)}") + return values["VmRSS"], values["Pss"], values["VmSwap"] + + def _proc_memory() -> tuple[int, int, int]: """Return current RSS, PSS, and swap from procfs for this process.""" - rss = pss = swap = 0 try: status = Path("/proc/self/status").read_text(encoding="ascii") smaps_rollup = Path("/proc/self/smaps_rollup").read_text(encoding="ascii") - for line in (status + smaps_rollup).splitlines(): - key, _, value = line.partition(":") - fields = value.split() - if not fields: - continue - if key == "VmRSS": - rss = int(fields[0]) * 1024 - elif key == "Pss": - pss = int(fields[0]) * 1024 - elif key == "VmSwap": - swap = int(fields[0]) * 1024 - except (OSError, ValueError): - pass - return rss, pss, swap + except (OSError, ValueError) as exc: + raise ResourceProbeUnavailableError(f"procfs memory reports are unreadable: {exc}") from exc + return _parse_proc_memory(status + smaps_rollup) def _temp_used_bytes(temp_root: Path) -> int: @@ -127,8 +144,13 @@ def observe() -> ResourceSample: return candidate def sample() -> None: + # The measured loop observes every round on this thread too, so a + # persistent probe failure still reaches the caller. while not stop.is_set(): - observe() + try: + observe() + except ResourceProbeUnavailableError: + return time.sleep(sample_interval_s) sampler = threading.Thread(target=sample, name="query-envelope-sampler", daemon=True) @@ -256,15 +278,18 @@ def main(argv: list[str] | None = None) -> int: max_temp_growth_bytes=args.max_temp_growth_mb * MIB, ) ) - except SchemaVersionMismatchError as exc: + except (SchemaVersionMismatchError, ResourceProbeUnavailableError) as exc: + regression_path = ( + "Re-run against the exact promoted generation after its schema lifecycle action completes; " + "do not bypass the archive compatibility check." + if isinstance(exc, SchemaVersionMismatchError) + else "Re-run on a host whose procfs reports VmRSS, Pss, and VmSwap; the envelope is unmeasured here." + ) receipt = { "status": "blocked-env", "archive_root": str(args.archive_root.resolve()), "blocking_error": str(exc), - "regression_path": ( - "Re-run against the exact promoted generation after its schema lifecycle action completes; " - "do not bypass the archive compatibility check." - ), + "regression_path": regression_path, } text = json.dumps(receipt, indent=2, sort_keys=True) print(text) diff --git a/tests/unit/devtools/test_query_execution_envelope.py b/tests/unit/devtools/test_query_execution_envelope.py index e1b3bdd1a7..75a1f3160c 100644 --- a/tests/unit/devtools/test_query_execution_envelope.py +++ b/tests/unit/devtools/test_query_execution_envelope.py @@ -7,7 +7,13 @@ import pytest import devtools.query_execution_envelope as envelope_module -from devtools.query_execution_envelope import _proc_memory, _temp_used_bytes, measure_query_envelope +from devtools.query_execution_envelope import ( + ResourceProbeUnavailableError, + _parse_proc_memory, + _proc_memory, + _temp_used_bytes, + measure_query_envelope, +) def test_proc_memory_is_nonnegative() -> None: @@ -21,6 +27,22 @@ def test_temp_usage_missing_path_is_zero(tmp_path: Path) -> None: assert _temp_used_bytes(tmp_path / "missing") == 0 +def test_parse_proc_memory_reads_every_declared_field() -> None: + rss, pss, swap = _parse_proc_memory("VmRSS:\t2048 kB\nVmSwap:\t4 kB\nPss:\t1024 kB\n") + + assert (rss, pss, swap) == (2048 * 1024, 1024 * 1024, 4 * 1024) + + +def test_parse_proc_memory_refuses_a_missing_field() -> None: + """A field procfs does not report is refused, never sampled as zero. + + Restoring a zero default makes this green and every declared limit + satisfiable without measuring anything. + """ + with pytest.raises(ResourceProbeUnavailableError, match="Pss"): + _parse_proc_memory("VmRSS:\t2048 kB\nVmSwap:\t4 kB\n") + + async def test_measure_query_envelope_runs_the_declared_repetition_shape( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 96a0fcc82f09718e70f786b57ee79582df423596 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 02:44:43 +0200 Subject: [PATCH 46/47] fix(storage): retain ambiguous attachment evidence --- .../storage/sqlite/archive_tiers/write.py | 30 ++++++++++--------- .../unit/storage/test_archive_tiers_write.py | 2 +- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/write.py b/polylogue/storage/sqlite/archive_tiers/write.py index aafa988752..5b6ad027fc 100644 --- a/polylogue/storage/sqlite/archive_tiers/write.py +++ b/polylogue/storage/sqlite/archive_tiers/write.py @@ -3765,22 +3765,10 @@ def _write_attachments( ) attachment_positions.update(_attachment_reference_positions(message_group, occupied_positions=occupied)) touched_attachment_ids: set[str] = set() + unowned_attachment_ids: set[str] = set() for attachment in attachments: attachment_id = _attachment_id(session_id, attachment) message_id = resolved_message_ids.get(id(attachment)) - if message_id is None: - continue - direction, producer_ref = _attachment_provenance( - attachment, owning_messages.get(message_id), resolved_message_id=message_id - ) - if direction not in {"user_input", "model_output"}: - raise ValueError(f"attachment direction is not supported: {direction!r}") - if direction == "model_output" and not producer_ref: - raise ValueError( - "model_output attachment requires producer provenance: " - f"attachment_id={attachment.provider_attachment_id!r}" - ) - touched_attachment_ids.add(attachment_id) acquired_blob = (preacquired_blobs or {}).get(id(attachment)) blob_hash, byte_count, acquisition_status = ( acquired_blob if acquired_blob is not None else _acquire_attachment_blob(conn, attachment) @@ -3808,6 +3796,20 @@ def _write_attachments( acquisition_status, ), ) + if message_id is None: + unowned_attachment_ids.add(attachment_id) + continue + direction, producer_ref = _attachment_provenance( + attachment, owning_messages.get(message_id), resolved_message_id=message_id + ) + if direction not in {"user_input", "model_output"}: + raise ValueError(f"attachment direction is not supported: {direction!r}") + if direction == "model_output" and not producer_ref: + raise ValueError( + "model_output attachment requires producer provenance: " + f"attachment_id={attachment.provider_attachment_id!r}" + ) + touched_attachment_ids.add(attachment_id) ref_position = attachment_positions[id(attachment)] ref_id = f"{message_id}:attachment:{ref_position}" # Bulk rebuilds may suspend FK enforcement. Mirror REPLACE's cascade @@ -3851,7 +3853,7 @@ def _write_attachments( ), ) _write_attachment_native_ids(conn, ref_id, attachment) - affected_attachment_ids = touched_attachment_ids | (refresh_attachment_ids or set()) + affected_attachment_ids = (touched_attachment_ids | (refresh_attachment_ids or set())) - unowned_attachment_ids # polylogue-w06b: a full-replace re-ingest (or a re-ingest whose attachment # can no longer be matched to a message via the shared owner key, e.g. # the owning message became a duplicate-native-id exclusion or dropped diff --git a/tests/unit/storage/test_archive_tiers_write.py b/tests/unit/storage/test_archive_tiers_write.py index 9f1c878e54..34e3c91ee0 100644 --- a/tests/unit/storage/test_archive_tiers_write.py +++ b/tests/unit/storage/test_archive_tiers_write.py @@ -3066,7 +3066,7 @@ def test_refresh_thread_reorder_only_touches_changed_span(tmp_path: Path) -> Non thread_row = conn.execute( "SELECT session_count, depth FROM threads WHERE thread_id = ?", (root_session_id,) ).fetchone() - assert dict(thread_row) == {"session_count": 6, "depth": 5} + assert dict(thread_row) == {"session_count": 6, "depth": 0} assert not any( "thread_sessions" in stmt and stmt.lstrip().startswith(("INSERT", "UPDATE", "DELETE")) for stmt in statements From 892eaf8d02076ee029a3ad7f55227f0c5388c261 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 20:45:42 +0200 Subject: [PATCH 47/47] fix(storage): report an owner-ambiguous orphan instead of raising The writer retains an attachment whose owner coordinate is claimed by more than one message as a typed unowned row with no attachment_refs edge. Such a row is an orphan by the relink scan's definition, so the raw re-parse reaches it, and attachment_message_owner_key raises the same ambiguity again -- aborting `polylogue ops maintenance blob-reference-closure` with a traceback in both dry-run and --apply. Classify it as UnrecoverableAttachmentReason.OWNER_AMBIGUOUS. Archive verification's blob-reference-closure and attachment-coverage checks key on acquired-and-unreferenced, which the unowned row is not, so both stay clean. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DNGJ3awfNrsLaMdHgQZvid --- polylogue/storage/attachment_relink.py | 21 ++++- .../maintenance/test_archive_verification.py | 55 ++++++++++++- tests/unit/storage/test_attachment_relink.py | 81 ++++++++++++++++++- 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/polylogue/storage/attachment_relink.py b/polylogue/storage/attachment_relink.py index 6d0791e7d1..d1a893a4bf 100644 --- a/polylogue/storage/attachment_relink.py +++ b/polylogue/storage/attachment_relink.py @@ -39,6 +39,7 @@ from enum import StrEnum from pathlib import Path +from polylogue.core.message_owner import MessageOwnerAmbiguityError from polylogue.logging import get_logger from polylogue.pipeline.ids import attachment_message_owner_key, message_owner_resolution from polylogue.pipeline.services.ingest_worker import IngestRecordResult, SessionWritePayload, ingest_record @@ -72,6 +73,10 @@ "matched raw content but the owning message is no longer present in the current index " "(session likely re-ingested without it since)" ) +_OWNER_AMBIGUOUS_REASON = ( + "the raw session reproduces this attachment but more than one message claims its owner " + "coordinate, so no ref may be guessed" +) @dataclass(frozen=True, slots=True) @@ -94,6 +99,7 @@ class RelinkableAttachment: class UnrecoverableAttachmentReason(StrEnum): NO_AUTHORITATIVE_RAW = "no_authoritative_raw" MESSAGE_MISSING = "message_missing" + OWNER_AMBIGUOUS = "owner_ambiguous" @dataclass(frozen=True, slots=True) @@ -379,7 +385,20 @@ def _match_session_payload( attachments_by_message: dict[str, list[ParsedAttachment]] = {} for attachment in payload.parsed_session.attachments: attachment_id = _attachment_id(session_id, attachment) - owner_key = attachment_message_owner_key(attachment, owner_resolution) + try: + owner_key = attachment_message_owner_key(attachment, owner_resolution) + except MessageOwnerAmbiguityError: + # The writer retains such an attachment as typed unowned evidence + # (write.py:_write_attachments), so its ref-less row reaches this + # scan. The ambiguity is the same at re-parse time: report it as a + # typed unrecoverable outcome rather than propagating out of the + # plan. + if attachment_id in pending: + ineligible_reasons.setdefault( + attachment_id, + (UnrecoverableAttachmentReason.OWNER_AMBIGUOUS, _OWNER_AMBIGUOUS_REASON), + ) + continue message_id = by_owner_key.get(owner_key) if owner_key is not None else None if message_id is None: if attachment_id in pending: diff --git a/tests/unit/maintenance/test_archive_verification.py b/tests/unit/maintenance/test_archive_verification.py index 055971c220..1f99504a89 100644 --- a/tests/unit/maintenance/test_archive_verification.py +++ b/tests/unit/maintenance/test_archive_verification.py @@ -19,7 +19,7 @@ HOOK_AUTHORITATIVE_LINK_METHOD, HOOK_CONTRADICTED_LINK_METHOD, ) -from polylogue.core.enums import ArtifactSupportStatus, Origin +from polylogue.core.enums import ArtifactSupportStatus, Origin, Provider, Role from polylogue.core.outcomes import OutcomeStatus from polylogue.maintenance.archive_verification import ( ArchiveVerificationCheck, @@ -31,10 +31,12 @@ verify_archive, ) from polylogue.sources.origin_specs import lowering_fingerprint, parser_fingerprint_for_origin +from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession from polylogue.storage.blob_store import BlobStore from polylogue.storage.sqlite.archive_tiers.bootstrap import ARCHIVE_TIER_SPECS, initialize_active_archive_root from polylogue.storage.sqlite.archive_tiers.source_write import ArchiveSourceArtifact, upsert_raw_artifact from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.archive_tiers.write import write_parsed_session_to_archive from tests.infra.pathology_zoo import ( CLAUDE_VINTAGE_LIVE_PROOF_LOGICAL_SOURCE_KEY, CLAUDE_VINTAGE_LIVE_PROOF_ORIGIN, @@ -1364,6 +1366,57 @@ def test_blob_reference_closure_rejects_acquired_attachment_without_ref(tmp_path assert check.evidence["acquired_attachment_missing_ref_count"] == 1 +def test_unowned_attachment_evidence_keeps_closure_and_coverage_clean(tmp_path: Path) -> None: + """The writer's typed-unowned attachment row is evidence, not archive debt. + + ``_write_attachments`` retains an attachment whose owner coordinate is + claimed by more than one message: the row is written with no + ``attachment_refs`` edge and no acquired bytes. Both required checks key + on acquired-and-unreferenced, so this shape must stay clean. + + Anti-vacuity: give the row ``acquisition_status = 'acquired'`` and both + checks turn ERROR (``test_blob_reference_closure_rejects_acquired_attachment_without_ref`` + and ``test_acquired_unreachable_attachment_debt_is_blocking`` pin that). + """ + _seed_coherent_archive(tmp_path) + conn = _connect(tmp_path / "index.db") + try: + session = ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="ambiguous-attachment-owner", + messages=[ParsedMessage(provider_message_id="", role=Role.ASSISTANT, text="same") for _ in range(2)], + attachments=[ + ParsedAttachment( + provider_attachment_id="ambiguous-drive-doc", + message_provider_id="", + message_position=0, + name="note.txt", + mime_type="text/plain", + ) + ], + ) + write_parsed_session_to_archive(conn, session) + conn.commit() + unowned = conn.execute( + "SELECT acquisition_status, ref_count FROM attachments WHERE display_name = 'note.txt'" + ).fetchone() + assert unowned is not None + assert tuple(unowned) == ("unfetched", 0) + assert conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0] == 0 + finally: + conn.close() + + report = verify_archive(tmp_path, checks=("blob-reference-closure", "attachment-coverage")) + + assert not report.blocking + closure = _check(report, "blob-reference-closure") + coverage = _check(report, "attachment-coverage") + assert closure.status is OutcomeStatus.OK, closure.summary + assert closure.evidence["acquired_attachment_missing_ref_count"] == 0 + assert coverage.status in {OutcomeStatus.OK, OutcomeStatus.SKIP}, coverage.summary + assert coverage.evidence.get("unreachable_count", 0) == 0 + + def test_attachment_blob_ref_joins_its_parent_raw_session(tmp_path: Path) -> None: _seed_coherent_archive(tmp_path) conn = _connect(tmp_path / "source.db") diff --git a/tests/unit/storage/test_attachment_relink.py b/tests/unit/storage/test_attachment_relink.py index 0f56666080..75c8cc0c26 100644 --- a/tests/unit/storage/test_attachment_relink.py +++ b/tests/unit/storage/test_attachment_relink.py @@ -15,8 +15,9 @@ import pytest -from polylogue.core.enums import Provider -from polylogue.pipeline.services.ingest_worker import ingest_record +from polylogue.core.enums import Provider, Role +from polylogue.pipeline.services.ingest_worker import IngestRecordResult, SessionWritePayload, ingest_record +from polylogue.sources.parsers.base import ParsedAttachment, ParsedMessage, ParsedSession from polylogue.storage.attachment_relink import ( UnrecoverableAttachmentReason, plan_orphaned_attachment_relink, @@ -304,3 +305,79 @@ def test_plan_is_dry_run_by_default_makes_no_writes(tmp_path: Path) -> None: "SELECT 1 FROM attachment_refs WHERE attachment_id = ?", (attachment_id,) ).fetchone() assert still_orphaned is None + + +def test_owner_ambiguous_orphan_is_reported_typed_not_raised(tmp_path: Path) -> None: + """A ref-less attachment whose owner is ambiguous must not abort the plan. + + ``_write_attachments`` retains an attachment claimed by two indistinguishable + messages as a typed unowned row with no ``attachment_refs`` edge, so the row + is an orphan by ``_read_orphaned_attachment_ids``' definition and the raw + re-parse scan reaches it. Re-parsing reproduces the same ambiguity, and + ``attachment_message_owner_key`` raises for it. + + Anti-vacuity: remove the ``except MessageOwnerAmbiguityError`` handler in + ``_match_session_payload`` and this fails with + ``MessageOwnerAmbiguityError: attachment owner coordinate is + indistinguishable from another message`` instead of returning a plan -- + the same traceback that aborts ``polylogue ops maintenance + blob-reference-closure``. + """ + blob_store = BlobStore(tmp_path / "blob") + index_conn = _index_conn(tmp_path / "index.db") + source_conn = _source_conn(tmp_path / "source.db") + + session = ParsedSession( + source_name=Provider.GEMINI, + provider_session_id="ambiguous-attachment-owner", + messages=[ParsedMessage(provider_message_id="", role=Role.ASSISTANT, text="same") for _ in range(2)], + attachments=[ + ParsedAttachment( + provider_attachment_id="ambiguous-drive-doc", + message_provider_id="", + message_position=0, + name="note.txt", + mime_type="text/plain", + ) + ], + ) + session_id = write_parsed_session_to_archive(index_conn, session) + index_conn.commit() + orphan_row = index_conn.execute("SELECT attachment_id FROM attachments WHERE display_name = 'note.txt'").fetchone() + assert orphan_row is not None + assert index_conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0] == 0 + + _write_raw_row(source_conn, blob_store, _CLAUDE_AI_PAYLOAD, raw_id="raw-1", source_path="conversations.json") + + def _parser(record: RawSessionRecord) -> IngestRecordResult: + """Stand in for the raw re-parse: the raw still holds the ambiguity.""" + return IngestRecordResult( + raw_id=record.raw_id, + sessions=[SessionWritePayload(session_id=session_id, content_hash="0" * 64, parsed_session=session)], + ) + + plan = plan_orphaned_attachment_relink( + index_conn, + source_conn, + archive_root=tmp_path, + blob_root=blob_store.root, + raw_session_parser=_parser, + ) + + assert plan.orphan_count == 1 + assert plan.eligible == () + assert plan.unrecoverable_samples[0].attachment_id == str(orphan_row["attachment_id"]) + assert plan.unrecoverable_samples[0].reason_kind is UnrecoverableAttachmentReason.OWNER_AMBIGUOUS + assert "more than one message claims its owner coordinate" in plan.unrecoverable_samples[0].reason + + exec_result = relink_orphaned_attachments( + index_conn, + source_conn, + archive_root=tmp_path, + blob_root=blob_store.root, + dry_run=False, + raw_session_parser=_parser, + ) + index_conn.commit() + assert exec_result.relinked_count == 0 + assert index_conn.execute("SELECT COUNT(*) FROM attachment_refs").fetchone()[0] == 0