From e4d53af095ac85c2230ad5f9b546677fc21f37c6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 02:36:26 +0200 Subject: [PATCH 01/10] feat(query): parse durable reference operands Problem: durable analysis objects need to compose without erasing their provenance through text expansion. What changed: - parse hand-split from query/query-run/result-set/cohort references into typed RefOperand nodes - expose reference lineage through explain and defer grain resolution to a narrow planner resolver seam - reject compatibility-selector lowering so unresolved references fail closed Ref polylogue-rxdo.6 Co-Authored-By: Codex --- polylogue/archive/query/expression.py | 196 +++++++++++++++++++++++- tests/unit/cli/test_query_expression.py | 58 +++++++ 2 files changed, 253 insertions(+), 1 deletion(-) diff --git a/polylogue/archive/query/expression.py b/polylogue/archive/query/expression.py index 531ebf8924..fc5a98ecfe 100644 --- a/polylogue/archive/query/expression.py +++ b/polylogue/archive/query/expression.py @@ -108,7 +108,7 @@ import re from dataclasses import dataclass, field, replace from difflib import get_close_matches -from typing import Any, Literal, cast +from typing import Any, Literal, Protocol, cast from lark import Lark, Token, Transformer, v_args from lark.exceptions import UnexpectedInput, VisitError @@ -161,6 +161,7 @@ normalize_retrieval_lane, ) from polylogue.core.enums import Origin +from polylogue.core.refs import ObjectRef from polylogue.errors import PolylogueError @@ -280,6 +281,103 @@ class QueryExpressionAST: clauses: tuple[_LexToken, ...] boolean_predicate: QueryPredicate | None = None + ref_operand: RefOperand | None = None + + +RelationGrain = Literal[ + "session", + "message", + "action", + "block", + "assertion", + "file", + "run", + "observed-event", + "context-snapshot", + "delegation", +] +RefEvaluationMode = Literal["re-evaluate", "retained", "resolver-defined"] + + +@dataclass(frozen=True) +class RefOperand: + """A provenance-preserving operand, never a textual query expansion.""" + + reference: ObjectRef + grain: RelationGrain | None = None + + @property + def evaluation_mode(self) -> RefEvaluationMode: + if self.reference.kind == "query": + return "re-evaluate" + if self.reference.kind in {"query-run", "result-set"}: + return "retained" + return "resolver-defined" + + def to_payload(self) -> dict[str, object]: + payload: dict[str, object] = { + "kind": "ref_operand", + "reference": self.reference.format(), + "reference_kind": self.reference.kind, + "evaluation_mode": self.evaluation_mode, + } + if self.grain is not None: + payload["grain"] = self.grain + return payload + + +@dataclass(frozen=True) +class ResolvedRefOperand: + """Planner resolution of a :class:`RefOperand` relation.""" + + operand: RefOperand + grain: RelationGrain + lineage: tuple[ObjectRef, ...] = () + + +class RefResolver(Protocol): + """Narrow planner seam for durable query/result/cohort relation lookup.""" + + def resolve_ref_operand(self, operand: RefOperand) -> ResolvedRefOperand: ... + + +class RefOperandCycleError(ExpressionCompileError): + """Raised when an operand would make the reference lineage cyclic.""" + + +def resolve_ref_operand( + operand: RefOperand, + resolver: RefResolver, + *, + ancestry: tuple[ObjectRef, ...] = (), +) -> ResolvedRefOperand: + """Resolve an operand and reject a cycle before relation materialization.""" + + ancestry_refs = {ref.format() for ref in ancestry} + if operand.reference.format() in ancestry_refs: + raise RefOperandCycleError( + f"reference cycle detected at {operand.reference.format()}; use a non-recursive query definition", + field="from", + ) + resolved = resolver.resolve_ref_operand(operand) + formatted = [ref.format() for ref in (*ancestry, *resolved.lineage, operand.reference)] + if len(formatted) != len(set(formatted)): + raise RefOperandCycleError( + f"reference cycle detected while resolving {operand.reference.format()}; use a non-recursive query definition", + field="from", + ) + return resolved + + +@dataclass(frozen=True) +class ReferenceQueryPipeline: + """A pipeline rooted in a durable reference whose grain is planner-resolved.""" + + operand: RefOperand + stages: tuple[str, ...] = () + + def to_payload(self) -> dict[str, object]: + return {"source": self.operand.to_payload(), "stages": list(self.stages)} @dataclass(frozen=True) @@ -1617,6 +1715,53 @@ def _split_pipeline_stages(expression: str) -> tuple[str, ...]: return tuple(stages) +_REF_OPERAND_KINDS: frozenset[str] = frozenset({"query", "query-run", "result-set", "cohort"}) + + +def _parse_ref_operand_stage(stage: str) -> RefOperand | None: + """Parse the hand-split ``from `` source stage. + + It intentionally lives outside the Lark grammar with other pipeline + stages: a colon-bearing result reference must not compete with the generic + ``FIELD_CLAUSE`` terminal. + """ + + normalized = stage.strip() + if not normalized.lower().startswith("from "): + return None + ref_text = normalized[5:].strip() + if not ref_text or any(char.isspace() for char in ref_text): + raise ExpressionCompileError("pipeline `from` requires one object reference", field="from") + try: + reference = ObjectRef.parse(ref_text) + except ValueError as exc: + raise ExpressionCompileError(f"invalid pipeline `from` reference: {ref_text!r}", field="from") from exc + if reference.kind not in _REF_OPERAND_KINDS: + supported = ", ".join(sorted(_REF_OPERAND_KINDS)) + raise ExpressionCompileError( + f"pipeline `from` requires a query, query-run, result-set, or cohort reference; got {reference.kind!r}. " + f"Supported kinds: {supported}", + field="from", + ) + return RefOperand(reference=reference) + + +def parse_reference_query_pipeline(expression: str) -> ReferenceQueryPipeline | None: + """Parse a ``from `` pipeline into a provenance-carrying AST node. + + Relation lookup is deliberately deferred to the planner, which supplies + the grain, retained-run availability, and durable query-edge lineage. + """ + + stages = _split_pipeline_stages(expression) + if not stages: + return None + operand = _parse_ref_operand_stage(stages[0]) + if operand is None: + return None + return ReferenceQueryPipeline(operand=operand, stages=stages[1:]) + + #: Canonical query units wired for the ``with `` projection clause. #: The clause is unit-agnostic by construction (the fetch/attach path drives off #: the descriptor registry), but only units listed here are validated as @@ -2311,6 +2456,9 @@ def parse_expression_ast(expression: str) -> QueryExpressionAST: expression, _with_units, _with_unit_fields = _split_with_projection_clause(expression) if not expression: return QueryExpressionAST(()) + reference_pipeline = parse_reference_query_pipeline(expression) + if reference_pipeline is not None: + return QueryExpressionAST((), ref_operand=reference_pipeline.operand) source_where = _parse_source_where_predicate(expression) if source_where is not None: return QueryExpressionAST((), boolean_predicate=source_where) @@ -2460,6 +2608,7 @@ def _ast_payload( clauses: tuple[QueryExpressionExplainClause, ...] = (), predicate: QueryPredicate | None = None, unit_source: QueryUnitSource | None = None, + reference_pipeline: ReferenceQueryPipeline | None = None, ) -> dict[str, object]: payload: dict[str, object] = {"entry": entry} if clauses: @@ -2490,6 +2639,8 @@ def _ast_payload( unit_payload["pipeline_stages"] = [stage.to_payload() for stage in unit_source.pipeline_stages] unit_payload["pipeline"] = unit_source.pipeline.to_payload() payload["unit_source"] = unit_payload + if reference_pipeline is not None: + payload["reference_pipeline"] = reference_pipeline.to_payload() return payload @@ -2549,6 +2700,34 @@ def explain_expression(expression: str) -> QueryExpressionExplanation: plan_description=plan_description, ), ) + reference_pipeline = parse_reference_query_pipeline(stripped) + if reference_pipeline is not None: + operand = reference_pipeline.operand + selected_units = (operand.grain or "unresolved-reference",) + execution_legs = ("durable-reference-relation",) + plan_description = ( + f"reference operand: {operand.reference.format()}", + f"reference evaluation: {operand.evaluation_mode}", + "relation grain: resolver required before materialization", + ) + lowerer = "reference-operand-to-planner-relation" + return QueryExpressionExplanation( + source_text=source_text, + clauses=(), + lowerer=lowerer, + lowered_spec=SessionQuerySpec(), + selected_units=selected_units, + execution_legs=execution_legs, + plan_description=plan_description, + ast=_ast_payload(entry="reference_pipeline", reference_pipeline=reference_pipeline), + lowering_plan={ + "lowerer": lowerer, + "selected_units": list(selected_units), + "execution_legs": list(execution_legs), + "plan_description": list(plan_description), + "reference_lineage": [operand.reference.format()], + }, + ) unit_source = parse_unit_source_expression(stripped) if unit_source is not None: lowered = ( @@ -3094,6 +3273,14 @@ def compile_expression(expression: str) -> SessionQuerySpec: # expression before parsing it (outside the Lark grammar, like ``|`` stages). expression, with_units, with_unit_fields = _split_with_projection_clause(expression) + reference_pipeline = parse_reference_query_pipeline(expression) + if reference_pipeline is not None: + raise ExpressionCompileError( + "pipeline `from` requires the reference-aware query planner; " + "a durable relation is never expanded into text at the compatibility selector boundary", + field="from", + ) + ast = parse_expression_ast(expression) if ast.boolean_predicate is not None: similar_text, residual_predicate = _extract_semantic_seed(ast.boolean_predicate) @@ -3242,6 +3429,11 @@ def build_session_terminal_pipeline( "QueryExpressionExplainClause", "QueryExpressionExplanation", "QueryExpressionAST", + "RefOperand", + "RefOperandCycleError", + "RefResolver", + "ReferenceQueryPipeline", + "ResolvedRefOperand", "QueryUnitPipeline", "QueryUnitCountStage", "QueryUnitGroupStage", @@ -3259,8 +3451,10 @@ def build_session_terminal_pipeline( "SessionQueryTerminalStage", "SessionTerminalAction", "_HAS_BOOL_MAP", + "parse_reference_query_pipeline", "parse_unit_source_expression", "parse_expression_ast", + "resolve_ref_operand", "split_with_clause", "split_with_projection_clause", "WITH_PROJECTION_SUPPORTED_UNITS", diff --git a/tests/unit/cli/test_query_expression.py b/tests/unit/cli/test_query_expression.py index d487af7950..aae908dd47 100644 --- a/tests/unit/cli/test_query_expression.py +++ b/tests/unit/cli/test_query_expression.py @@ -33,6 +33,9 @@ QueryUnitPipeline, QueryUnitSessionScopeStage, QueryUnitSource, + RefOperand, + RefOperandCycleError, + ResolvedRefOperand, UnsupportedSessionTerminalActionError, _CountRangeToken, _CountToken, @@ -45,7 +48,9 @@ compile_expression_into, explain_expression, parse_expression_ast, + parse_reference_query_pipeline, parse_unit_source_expression, + resolve_ref_operand, ) from polylogue.archive.query.metadata import query_unit_descriptors from polylogue.archive.query.predicate import ( @@ -60,6 +65,7 @@ QueryTextPredicate, ) from polylogue.archive.query.spec import SessionQuerySpec +from polylogue.core.refs import ObjectRef from polylogue.storage.runtime import MessageRecord # --------------------------------------------------------------------------- @@ -493,6 +499,58 @@ def test_explain_expression_reports_session_scoped_pipeline_stage(self) -> None: class TestBooleanQueryExpression: + def test_reference_pipeline_preserves_result_set_as_typed_operand(self) -> None: + pipeline = parse_reference_query_pipeline("from result-set:stable-set | group by model | count") + + assert pipeline is not None + assert pipeline.operand == RefOperand(reference=ObjectRef(kind="result-set", object_id="stable-set")) + assert pipeline.stages == ("group by model", "count") + + def test_reference_pipeline_bypasses_lark_field_clause_colon_ambiguity(self) -> None: + ast = parse_expression_ast("from result-set:stable-set | count") + + assert ast.ref_operand == RefOperand(reference=ObjectRef(kind="result-set", object_id="stable-set")) + assert ast.clauses == () + + def test_reference_explain_reports_durable_lineage_without_textual_expansion(self) -> None: + payload = explain_expression("from result-set:stable-set | group by model | count").to_payload() + + assert payload["ast"] == { + "entry": "reference_pipeline", + "reference_pipeline": { + "source": { + "kind": "ref_operand", + "reference": "result-set:stable-set", + "reference_kind": "result-set", + "evaluation_mode": "retained", + }, + "stages": ["group by model", "count"], + }, + } + assert cast(dict[str, Any], payload["lowering_plan"])["reference_lineage"] == ["result-set:stable-set"] + + def test_reference_pipeline_rejects_compatibility_text_lowering(self) -> None: + with pytest.raises(ExpressionCompileError, match="never expanded into text"): + compile_expression("from result-set:stable-set") + + def test_reference_resolver_rejects_a_cycle_before_materialization(self) -> None: + operand = RefOperand(reference=ObjectRef(kind="query", object_id="stable-query")) + + class _CyclicResolver: + def resolve_ref_operand(self, resolved_operand: RefOperand) -> ResolvedRefOperand: + return ResolvedRefOperand( + operand=resolved_operand, + grain="session", + lineage=(ObjectRef(kind="query", object_id="stable-query"),), + ) + + with pytest.raises(RefOperandCycleError, match="reference cycle"): + resolve_ref_operand(operand, _CyclicResolver()) + + def test_macro_token_is_not_reinterpreted_as_a_reference_operand(self) -> None: + assert parse_reference_query_pipeline("@saved") is None + assert _clauses("@saved") == [_TextToken(text="@saved", quoted=False, negated=False)] + def test_boolean_ast_exposes_predicate_tree(self) -> None: ast = parse_expression_ast("repo:polylogue OR origin:chatgpt-export") From d8c71f72df0d76ffbc1f73467300240de7c55bf2 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 02:41:12 +0200 Subject: [PATCH 02/10] fix(query): validate durable reference identities Problem: the parser must accept only the immutable query reference shapes owned by the analysis-provenance substrate. What changed: - validate query/query-run/result-set operands with canonical substrate constructors - recognize an unpiped from reference as a pipeline source - cover malformed query and query-run identities Ref polylogue-rxdo.6 Co-Authored-By: Codex --- polylogue/archive/query/expression.py | 21 +++++++++++++++++++++ tests/unit/cli/test_query_expression.py | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/polylogue/archive/query/expression.py b/polylogue/archive/query/expression.py index fc5a98ecfe..4344217a79 100644 --- a/polylogue/archive/query/expression.py +++ b/polylogue/archive/query/expression.py @@ -161,6 +161,7 @@ normalize_retrieval_lane, ) from polylogue.core.enums import Origin +from polylogue.core.query_identity import query_ref, query_run_ref, result_set_ref from polylogue.core.refs import ObjectRef from polylogue.errors import PolylogueError @@ -1718,6 +1719,20 @@ def _split_pipeline_stages(expression: str) -> tuple[str, ...]: _REF_OPERAND_KINDS: frozenset[str] = frozenset({"query", "query-run", "result-set", "cohort"}) +def _validate_ref_operand_identity(reference: ObjectRef) -> None: + """Apply the substrate's canonical identity constraints at the DSL edge.""" + + try: + if reference.kind == "query": + query_ref(reference.object_id) + elif reference.kind == "query-run": + query_run_ref(reference.object_id) + elif reference.kind == "result-set": + result_set_ref(reference.object_id) + except ValueError as exc: + raise ExpressionCompileError(f"invalid {reference.kind} reference: {exc}", field="from") from exc + + def _parse_ref_operand_stage(stage: str) -> RefOperand | None: """Parse the hand-split ``from `` source stage. @@ -1743,6 +1758,7 @@ def _parse_ref_operand_stage(stage: str) -> RefOperand | None: f"Supported kinds: {supported}", field="from", ) + _validate_ref_operand_identity(reference) return RefOperand(reference=reference) @@ -1754,6 +1770,11 @@ def parse_reference_query_pipeline(expression: str) -> ReferenceQueryPipeline | """ stages = _split_pipeline_stages(expression) + if not stages: + stripped = expression.strip() + if not stripped: + return None + stages = (stripped,) if not stages: return None operand = _parse_ref_operand_stage(stages[0]) diff --git a/tests/unit/cli/test_query_expression.py b/tests/unit/cli/test_query_expression.py index aae908dd47..4e2fe0a090 100644 --- a/tests/unit/cli/test_query_expression.py +++ b/tests/unit/cli/test_query_expression.py @@ -512,6 +512,17 @@ def test_reference_pipeline_bypasses_lark_field_clause_colon_ambiguity(self) -> assert ast.ref_operand == RefOperand(reference=ObjectRef(kind="result-set", object_id="stable-set")) assert ast.clauses == () + @pytest.mark.parametrize( + ("expression", "match"), + [ + ("from query:not-a-hash", "query hash must be 64 lowercase hexadecimal characters"), + ("from query-run:not-a-run", "query run id must start with 'qr_'"), + ], + ) + def test_reference_pipeline_uses_canonical_substrate_identity_validation(self, expression: str, match: str) -> None: + with pytest.raises(ExpressionCompileError, match=match): + parse_reference_query_pipeline(expression) + def test_reference_explain_reports_durable_lineage_without_textual_expansion(self) -> None: payload = explain_expression("from result-set:stable-set | group by model | count").to_payload() From 21b8f5a188b1d50ce05871f899d8d832216030a8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 07:15:55 +0200 Subject: [PATCH 03/10] feat(query): materialize watched query relations Problem: durable query identities were not versioned executable contracts, and watched-query convergence had no retained baseline or finding path. What changed: add the v8 definition/evaluation receipt and retained-run contracts, a planner-owned canonical-plan evaluator seam, durable RefOperand resolution, and the session-scoped standing-query stage. The stage establishes user-tier baselines before issuing candidate-only delta or expected-count drift findings. Compatibility/migration: existing query rows retain protocol v0; newly created canonical definitions bind v1. Saved-query migration runs after v8 installs the version column. Ref polylogue-rxdo.6 Ref polylogue-rxdo.5 Ref polylogue-rxdo.2 Co-Authored-By: Claude --- docs/plans/topology-target.yaml | 17 +- docs/topology-status.md | 8 +- polylogue/archive/query/evaluator.py | 159 ++++++++++ polylogue/archive/query/expression.py | 14 +- polylogue/core/query_identity.py | 11 + polylogue/daemon/convergence_stages.py | 277 +++++++++++++++++- .../storage/sqlite/archive_tiers/user.py | 33 ++- polylogue/storage/sqlite/migration_runner.py | 4 +- .../user/008_query_evaluation_contracts.sql | 33 +++ polylogue/storage/sqlite/query_objects.py | 254 +++++++++++++++- tests/unit/archive/query/test_evaluator.py | 103 +++++++ tests/unit/core/test_query_identity.py | 18 ++ tests/unit/daemon/test_convergence_stages.py | 2 +- tests/unit/daemon/test_standing_queries.py | 138 +++++++++ tests/unit/storage/test_durable_migrations.py | 6 +- tests/unit/storage/test_query_objects.py | 60 ++++ 16 files changed, 1106 insertions(+), 31 deletions(-) create mode 100644 polylogue/archive/query/evaluator.py create mode 100644 polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql create mode 100644 tests/unit/archive/query/test_evaluator.py create mode 100644 tests/unit/daemon/test_standing_queries.py diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index 8cc18e1784..735088b6fd 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -331,8 +331,13 @@ files: target: polylogue/archive/query/completions.py owner: archive-query reason: archive-domain query semantics + - path: polylogue/archive/query/evaluator.py + loc: 161 + target: polylogue/archive/query/evaluator.py + owner: archive-query + reason: archive-domain query semantics - path: polylogue/archive/query/expression.py - loc: 3270 + loc: 3497 target: polylogue/archive/query/expression.py owner: archive-query reason: archive-domain query semantics @@ -1224,7 +1229,7 @@ files: owner: core-primitive reason: core primitive - path: polylogue/core/query_identity.py - loc: 157 + loc: 168 target: polylogue/core/query_identity.py owner: core-primitive reason: core primitive @@ -1315,7 +1320,7 @@ files: target: polylogue/daemon/convergence_debt_status.py owner: stable - path: polylogue/daemon/convergence_stages.py - loc: 1674 + loc: 1942 target: polylogue/daemon/convergence_stages.py owner: stable - path: polylogue/daemon/cursor_lag_alert.py @@ -3525,7 +3530,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/types.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/user.py - loc: 211 + loc: 242 target: polylogue/storage/sqlite/archive_tiers/user.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/user_annotations.py @@ -3573,7 +3578,7 @@ files: target: polylogue/storage/sqlite/maintenance.py owner: stable - path: polylogue/storage/sqlite/migration_runner.py - loc: 569 + loc: 571 target: polylogue/storage/sqlite/migration_runner.py owner: stable - path: polylogue/storage/sqlite/migrations/__init__.py @@ -3769,7 +3774,7 @@ files: target: polylogue/storage/sqlite/queries/tool_usage.py owner: stable - path: polylogue/storage/sqlite/query_objects.py - loc: 242 + loc: 467 target: polylogue/storage/sqlite/query_objects.py owner: stable - path: polylogue/storage/sqlite/query_store.py diff --git a/docs/topology-status.md b/docs/topology-status.md index 461126eaf0..27fd1c5fed 100644 --- a/docs/topology-status.md +++ b/docs/topology-status.md @@ -19,7 +19,7 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe | archive-phase | — | 2 | 2 | 0 | 0 | | archive-projection | — | 5 | 5 | 0 | 0 | | archive-provider | — | 2 | 2 | 0 | 0 | -| archive-query | archive query semantics | 26 | 26 | 0 | 0 | +| archive-query | archive query semantics | 27 | 27 | 0 | 0 | | archive-raw-payload | — | 5 | 5 | 0 | 0 | | archive-semantic | — | 12 | 12 | 0 | 0 | | archive-session | — | 16 | 16 | 0 | 0 | @@ -28,12 +28,12 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe ### Summary -- **Stable** (no move scoped): 777 +- **Stable** (no move scoped): 774 - **Kernel** (polylogue/ root): 10 - **Primitives** (storage-root): 18 - **TBD** (cell needs explicit assignment): 6 -- **Total declared**: 930 -- **Realized polylogue/**/*.py**: 930 files declared +- **Total declared**: 928 +- **Realized polylogue/**/*.py**: 928 files declared ### TBD cells (require explicit routing) diff --git a/polylogue/archive/query/evaluator.py b/polylogue/archive/query/evaluator.py new file mode 100644 index 0000000000..b0125e0165 --- /dev/null +++ b/polylogue/archive/query/evaluator.py @@ -0,0 +1,159 @@ +"""Planner-owned execution seam for durable canonical query definitions. + +Canonical query identity is deliberately not an executable serialization. A +planner implementation receives its typed canonical definition here and owns +the language-version dispatch, temporal binding, and source access needed to +evaluate it. +""" + +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from typing import Literal, Protocol + +from polylogue.archive.query.expression import RefOperand, RelationGrain, ResolvedRefOperand +from polylogue.core.refs import ObjectRef +from polylogue.storage.sqlite.query_objects import ( + EvaluationReceipt, + QueryObject, + get_query, + get_result_set, + get_result_set_members, + get_retained_query_run, + put_query_edge, +) + +EvaluationPurpose = Literal["reference", "standing-watch", "finding-drift"] + + +class RetainedRelationUnavailableError(ValueError): + """A reference requires a durable relation that was never retained.""" + + +@dataclass(frozen=True, slots=True) +class QueryEvaluationRequest: + query: QueryObject + purpose: EvaluationPurpose + changed_session_ids: tuple[str, ...] = () + excluded_scope_refs: tuple[str, ...] = () + excluded_origin_prefixes: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class QueryEvaluation: + grain: RelationGrain + member_refs: tuple[str, ...] + corpus_epoch: str + exactness: Literal["exact", "capped", "sampled", "estimate"] + receipt: EvaluationReceipt + cache_only: bool = False + + +class CanonicalPlanEvaluator(Protocol): + """Evaluate canonical definitions without reverse-compiling identity JSON.""" + + def evaluate(self, request: QueryEvaluationRequest) -> QueryEvaluation: ... + + def resolve_cohort(self, operand: RefOperand) -> QueryEvaluation: ... + + +class DurableRefResolver: + """Resolve `from` operands through durable manifests and the planner seam.""" + + def __init__( + self, + conn: sqlite3.Connection, + evaluator: CanonicalPlanEvaluator, + *, + owner_query_hash: str | None = None, + created_at_ms: int = 0, + ) -> None: + self._conn = conn + self._evaluator = evaluator + self._owner_query_hash = owner_query_hash + self._created_at_ms = created_at_ms + + def resolve_ref_operand(self, operand: RefOperand) -> ResolvedRefOperand: + reference = operand.reference + if reference.kind == "query": + query = get_query(self._conn, reference.object_id) + if query is None: + raise KeyError(reference.format()) + evaluation = self._evaluator.evaluate(QueryEvaluationRequest(query=query, purpose="reference")) + self._record_operand_edge(query.query_hash) + return ResolvedRefOperand( + operand=operand, + grain=evaluation.grain, + lineage=(query_ref_for(query),), + member_refs=evaluation.member_refs, + ) + if reference.kind == "query-run": + retained = get_retained_query_run(self._conn, reference.object_id) + if retained is None: + raise RetainedRelationUnavailableError( + f"query run {reference.format()} has no retained relation; use query: to re-evaluate" + ) + return self._retained_result(operand, retained.result_set_id, extra_lineage=(reference,)) + if reference.kind == "result-set": + return self._retained_result(operand, reference.object_id) + if reference.kind == "cohort": + evaluation = self._evaluator.resolve_cohort(operand) + return ResolvedRefOperand( + operand=operand, + grain=evaluation.grain, + lineage=(reference,), + member_refs=evaluation.member_refs, + ) + raise ValueError(f"unsupported ref operand kind: {reference.kind}") + + def _retained_result( + self, + operand: RefOperand, + result_set_id: str, + *, + extra_lineage: tuple[ObjectRef, ...] = (), + ) -> ResolvedRefOperand: + manifest = get_result_set(self._conn, result_set_id) + if manifest is None: + raise RetainedRelationUnavailableError(f"retained result-set:{result_set_id} is unavailable") + members = get_result_set_members(self._conn, result_set_id) + if manifest.member_count != len(members): + raise RetainedRelationUnavailableError( + f"result-set:{result_set_id} has no retained exact relation; use query: to re-evaluate" + ) + query = get_query(self._conn, manifest.query_hash) + lineage = (*extra_lineage, ObjectRef(kind="result-set", object_id=result_set_id)) + if query is not None: + lineage = (*lineage, query_ref_for(query)) + return ResolvedRefOperand( + operand=operand, + grain=manifest.grain, # type: ignore[arg-type] + lineage=lineage, + member_refs=members, + ) + + def _record_operand_edge(self, target_query_hash: str) -> None: + if self._owner_query_hash is None: + return + put_query_edge( + self._conn, + src_query_hash=self._owner_query_hash, + dst_query_hash=target_query_hash, + edge_kind="operand-of", + created_at_ms=self._created_at_ms, + ) + + +def query_ref_for(query: QueryObject) -> ObjectRef: + return ObjectRef(kind="query", object_id=query.query_hash) + + +__all__ = [ + "CanonicalPlanEvaluator", + "DurableRefResolver", + "EvaluationPurpose", + "QueryEvaluation", + "QueryEvaluationRequest", + "RetainedRelationUnavailableError", +] diff --git a/polylogue/archive/query/expression.py b/polylogue/archive/query/expression.py index 4344217a79..fe0bb3c80b 100644 --- a/polylogue/archive/query/expression.py +++ b/polylogue/archive/query/expression.py @@ -334,6 +334,7 @@ class ResolvedRefOperand: operand: RefOperand grain: RelationGrain lineage: tuple[ObjectRef, ...] = () + member_refs: tuple[str, ...] = () class RefResolver(Protocol): @@ -361,7 +362,18 @@ def resolve_ref_operand( field="from", ) resolved = resolver.resolve_ref_operand(operand) - formatted = [ref.format() for ref in (*ancestry, *resolved.lineage, operand.reference)] + if operand.grain is not None and operand.grain != resolved.grain: + raise ExpressionCompileError( + f"reference {operand.reference.format()} has grain {resolved.grain!r}, not {operand.grain!r}; " + "use a same-grain relation or an explicit grain-changing projection", + field="from", + ) + # Resolvers may include the direct operand in lineage for explainability; + # add it only when absent so a normal one-hop query ref is not mistaken for + # a cycle. Repeated ancestors or indirect lineage still fail closed. + formatted = [ref.format() for ref in (*ancestry, *resolved.lineage)] + if operand.reference.format() not in formatted: + formatted.append(operand.reference.format()) if len(formatted) != len(set(formatted)): raise RefOperandCycleError( f"reference cycle detected while resolving {operand.reference.format()}; use a non-recursive query definition", diff --git a/polylogue/core/query_identity.py b/polylogue/core/query_identity.py index 86d51974da..103c49d205 100644 --- a/polylogue/core/query_identity.py +++ b/polylogue/core/query_identity.py @@ -28,6 +28,8 @@ QUERY_RUN_REF_KIND: Final = "query-run" RESULT_SET_REF_KIND: Final = "result-set" QUERY_RUN_ID_PREFIX: Final = "qr_" +LEGACY_QUERY_DEFINITION_PROTOCOL_VERSION: Final = "polylogue.query-definition.v0" +QUERY_DEFINITION_PROTOCOL_VERSION: Final = "polylogue.query-definition.v1" _COMMUTATIVE_OPERATORS: Final = frozenset({"and", "or"}) _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$") @@ -43,6 +45,7 @@ def canonical_query_plan( lane: str, rank_policy: str, field_aliases: Mapping[str, str] | None = None, + definition_protocol_version: str = QUERY_DEFINITION_PROTOCOL_VERSION, ) -> dict[str, JsonValue]: """Return the stable canonical payload used to identify a query. @@ -51,8 +54,12 @@ def canonical_query_plan( field token; it is applied only to values in ``field`` keys. """ aliases = {_nfc(key): _nfc(value) for key, value in (field_aliases or {}).items()} + protocol_version = _nfc(definition_protocol_version) + if not protocol_version: + raise ValueError("definition protocol version cannot be empty") return { "ast": _canonical_value(planned_ast, field_aliases=aliases), + "definition_protocol_version": protocol_version, "grain": _nfc(grain), "lane": _nfc(lane), "rank_policy": _nfc(rank_policy), @@ -66,6 +73,7 @@ def query_hash_for_plan( lane: str, rank_policy: str, field_aliases: Mapping[str, str] | None = None, + definition_protocol_version: str = QUERY_DEFINITION_PROTOCOL_VERSION, ) -> str: """Return the SHA-256 identity of an expanded planned query.""" return hash_payload( @@ -75,6 +83,7 @@ def query_hash_for_plan( lane=lane, rank_policy=rank_policy, field_aliases=field_aliases, + definition_protocol_version=definition_protocol_version, ) ) @@ -145,6 +154,8 @@ def _require_sha256(value: str, *, label: str) -> None: __all__ = [ "JsonValue", + "LEGACY_QUERY_DEFINITION_PROTOCOL_VERSION", + "QUERY_DEFINITION_PROTOCOL_VERSION", "QUERY_REF_KIND", "QUERY_RUN_ID_PREFIX", "QUERY_RUN_REF_KIND", diff --git a/polylogue/daemon/convergence_stages.py b/polylogue/daemon/convergence_stages.py index 37a59b7705..41f8281edc 100644 --- a/polylogue/daemon/convergence_stages.py +++ b/polylogue/daemon/convergence_stages.py @@ -15,12 +15,16 @@ import sqlite3 import time -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING +from polylogue.archive.query.evaluator import CanonicalPlanEvaluator, QueryEvaluation, QueryEvaluationRequest from polylogue.config import load_polylogue_config +from polylogue.core.enums import AssertionKind, AssertionStatus +from polylogue.core.hashing import hash_payload +from polylogue.core.query_identity import query_ref, result_set_ref from polylogue.daemon.convergence import ConvergenceStage, StageExecuteReturn, StageExecutionResult from polylogue.logging import get_logger from polylogue.storage.runtime import SESSION_INSIGHT_MATERIALIZER_VERSION @@ -28,7 +32,21 @@ session_ids_for_source_path, session_ids_for_source_paths, ) +from polylogue.storage.sqlite.archive_tiers.user_write import ( + FindingAssertion, + list_assertion_claims, + upsert_findings_as_assertions, +) from polylogue.storage.sqlite.connection_profile import open_daemon_connection +from polylogue.storage.sqlite.query_objects import ( + get_latest_result_set, + get_query, + get_result_set, + list_watched_queries, + membership_merkle_root, + put_evaluation_receipt, + put_result_set, +) if TYPE_CHECKING: pass @@ -553,12 +571,268 @@ def execute_sessions(session_ids: Sequence[str]) -> StageExecuteReturn: ) +# ── Stage: standing queries ─────────────────────────────────────── + + +def make_standing_query_stage( + db_path: Path, + *, + evaluator: CanonicalPlanEvaluator | None = None, +) -> ConvergenceStage: + """Re-evaluate watched definitions after affected session convergence. + + The canonical-plan evaluator is deliberately injected. Identity JSON is + not an executable plan, so the stage stays inert until the owning planner + is supplied by broad runtime wiring. Its durable baseline, receipt, and + candidate-finding semantics are nevertheless the production path. + """ + + def check(_path: Path) -> bool: + return False + + def execute(_path: Path) -> StageExecuteReturn: + return True + + def check_sessions(session_ids: Sequence[str]) -> set[str]: + if evaluator is None or not session_ids: + return set() + user_db = _standing_user_db_path(db_path) + if not user_db.exists(): + return set() + try: + with sqlite3.connect(f"file:{user_db}?mode=ro", uri=True, timeout=5.0) as conn: + return set(session_ids) if list_watched_queries(conn) else set() + except Exception: + logger.warning("standing-queries: watch lookup failed", exc_info=True) + return set(session_ids) + + def execute_sessions(session_ids: Sequence[str]) -> StageExecuteReturn: + if evaluator is None: + return True + user_db = _standing_user_db_path(db_path) + if not user_db.exists(): + return True + now_ms = int(time.time() * 1000) + ids = tuple(dict.fromkeys(str(session_id) for session_id in session_ids if session_id)) + try: + conn = open_daemon_connection(user_db, timeout=30.0) + try: + watched = list_watched_queries(conn) + for query in watched: + query_reference = query_ref(query.query_hash).format() + evaluation = evaluator.evaluate( + QueryEvaluationRequest( + query=query, + purpose="standing-watch", + changed_session_ids=ids, + excluded_scope_refs=(query_reference,), + excluded_origin_prefixes=("notice.",), + ) + ) + if evaluation.cache_only: + # An index-only relation after reset is not evidence of + # membership drift. Compare durable user-tier baselines only. + continue + _materialize_watch_evaluation(conn, query.query_hash, evaluation, now_ms=now_ms) + _materialize_promoted_finding_drifts(conn, evaluator, now_ms=now_ms) + conn.commit() + finally: + conn.close() + return True + except Exception: + logger.warning("standing-queries: evaluation deferred", exc_info=True) + return False + + return ConvergenceStage( + name="standing-queries", + description="Re-evaluate watched query definitions and emit candidate deltas", + check=check, + execute=execute, + check_sessions=check_sessions, + execute_sessions=execute_sessions, + cpu_bound=False, + false_means_pending=True, + ) + + +def _standing_user_db_path(db_path: Path) -> Path: + archive_db = _active_archive_index_path(db_path) + base = archive_db if archive_db is not None else db_path + return base.with_name("user.db") + + +def _watch_result_set_id(query_hash: str, member_refs: tuple[str, ...]) -> str: + return f"watch-{hash_payload((query_hash, membership_merkle_root(member_refs)))}" + + +def _materialize_watch_evaluation( + conn: sqlite3.Connection, + query_hash: str, + evaluation: QueryEvaluation, + *, + now_ms: int, +) -> None: + baseline = get_latest_result_set(conn, query_hash=query_hash, persistence_class="watch") + root = membership_merkle_root(evaluation.member_refs) + if baseline is not None and baseline.membership_merkle_root == root: + put_evaluation_receipt( + conn, + query_hash=query_hash, + receipt=evaluation.receipt, + result_set_id=baseline.result_set_id, + created_at_ms=now_ms, + ) + return + result_set_id = _watch_result_set_id(query_hash, evaluation.member_refs) + current = get_result_set(conn, result_set_id) + if current is None: + current = put_result_set( + conn, + result_set_id=result_set_id, + query_hash=query_hash, + grain=evaluation.grain, + corpus_epoch=evaluation.corpus_epoch, + member_refs=evaluation.member_refs, + exactness=evaluation.exactness, + persistence_class="watch", + created_at_ms=now_ms, + ) + put_evaluation_receipt( + conn, + query_hash=query_hash, + receipt=evaluation.receipt, + result_set_id=current.result_set_id, + created_at_ms=now_ms, + ) + if baseline is None: + return + query_reference = query_ref(query_hash).format() + current_reference = result_set_ref(current.result_set_id).format() + baseline_reference = result_set_ref(baseline.result_set_id).format() + upsert_findings_as_assertions( + conn, + [ + FindingAssertion( + claim_key="standing-query-membership-delta", + target_ref=query_reference, + body_text="Watched query membership changed after archive convergence.", + finding_kind="query-delta", + statistic={"op": "count", "value": current.member_count, "unit": "members"}, + n=current.member_count, + query_ref=query_reference, + result_set_ref=current_reference, + baseline_ref=baseline_reference, + current_ref=current_reference, + detector_ref="agent:standing-query-detector.v1", + scope_ref=query_reference, + ) + ], + now_ms=now_ms, + ) + + +def _materialize_promoted_finding_drifts( + conn: sqlite3.Connection, + evaluator: CanonicalPlanEvaluator, + *, + now_ms: int, +) -> None: + """Emit a new candidate when an accepted expected-count finding diverges.""" + for finding in list_assertion_claims( + conn, + kinds=(AssertionKind.FINDING,), + statuses=(AssertionStatus.ACCEPTED,), + ): + value = finding.value if isinstance(finding.value, dict) else {} + expected = value.get("expected") + query_reference = value.get("query_ref") + if not isinstance(expected, dict) or not isinstance(query_reference, str): + continue + query_hash = query_reference.removeprefix("query:") + query = get_query(conn, query_hash) + if query is None: + continue + evaluation = evaluator.evaluate( + QueryEvaluationRequest( + query=query, + purpose="finding-drift", + excluded_scope_refs=(f"assertion:{finding.assertion_id}",), + excluded_origin_prefixes=("notice.",), + ) + ) + if evaluation.cache_only or _matches_expected_count(expected, len(evaluation.member_refs)): + continue + current_id = f"finding-{hash_payload((query_hash, membership_merkle_root(evaluation.member_refs)))}" + current = get_result_set(conn, current_id) + if current is None: + current = put_result_set( + conn, + result_set_id=current_id, + query_hash=query_hash, + grain=evaluation.grain, + corpus_epoch=evaluation.corpus_epoch, + member_refs=evaluation.member_refs, + exactness=evaluation.exactness, + persistence_class="finding", + created_at_ms=now_ms, + ) + put_evaluation_receipt( + conn, + query_hash=query_hash, + receipt=evaluation.receipt, + result_set_id=current.result_set_id, + created_at_ms=now_ms, + ) + current_reference = result_set_ref(current.result_set_id).format() + upsert_findings_as_assertions( + conn, + [ + FindingAssertion( + claim_key="promoted-finding-expected-count-drift", + target_ref=f"assertion:{finding.assertion_id}", + body_text="Promoted finding no longer matches its expected member count.", + finding_kind="query-drift", + statistic={"op": "count", "value": current.member_count, "unit": "members"}, + n=current.member_count, + query_ref=query_reference, + result_set_ref=current_reference, + current_ref=current_reference, + expected=expected, + detector_ref="agent:standing-query-detector.v1", + scope_ref=f"assertion:{finding.assertion_id}", + ) + ], + now_ms=now_ms, + ) + + +def _matches_expected_count(expected: Mapping[str, object], actual: int) -> bool: + if expected.get("measure") != "member_count": + return True + value = expected.get("value") + if isinstance(value, bool) or not isinstance(value, int): + return True + op = expected.get("op") + if not isinstance(op, str): + return True + comparisons = { + "=": actual == value, + "!=": actual != value, + ">": actual > value, + ">=": actual >= value, + "<": actual < value, + "<=": actual <= value, + } + return comparisons.get(op, True) + + def make_default_convergence_stages(db_path: Path) -> tuple[ConvergenceStage, ...]: """Build the daemon's default post-ingest convergence stage set.""" return ( make_fts_stage(db_path), make_embed_stage(db_path), make_insights_stage(db_path), + make_standing_query_stage(db_path), ) @@ -1671,4 +1945,5 @@ def _archive_insights_execute_ids(conn: sqlite3.Connection, session_ids: Sequenc "make_embed_stage", "make_fts_stage", "make_insights_stage", + "make_standing_query_stage", ] diff --git a/polylogue/storage/sqlite/archive_tiers/user.py b/polylogue/storage/sqlite/archive_tiers/user.py index 25f78a986b..c143427437 100644 --- a/polylogue/storage/sqlite/archive_tiers/user.py +++ b/polylogue/storage/sqlite/archive_tiers/user.py @@ -2,7 +2,7 @@ from __future__ import annotations -USER_SCHEMA_VERSION = 7 +USER_SCHEMA_VERSION = 8 USER_DDL = """ -- Unified evidence-linked user assertion. Marks, annotations, @@ -50,6 +50,7 @@ grain TEXT NOT NULL CHECK(length(trim(grain)) > 0), lane TEXT NOT NULL CHECK(length(trim(lane)) > 0), rank_policy TEXT NOT NULL CHECK(length(trim(rank_policy)) > 0), + definition_protocol_version TEXT NOT NULL CHECK(length(trim(definition_protocol_version)) > 0), created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) ) STRICT; @@ -57,12 +58,16 @@ name TEXT PRIMARY KEY NOT NULL CHECK(length(trim(name)) > 0), query_hash TEXT NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, supersedes_query_hash TEXT REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, + watch INTEGER NOT NULL DEFAULT 0 CHECK(watch IN (0, 1)), updated_at_ms INTEGER NOT NULL CHECK(updated_at_ms >= 0) ) STRICT; CREATE INDEX IF NOT EXISTS idx_query_names_query_hash ON query_names(query_hash, updated_at_ms DESC); +CREATE INDEX IF NOT EXISTS idx_query_names_watch +ON query_names(watch, updated_at_ms DESC, name); + CREATE TABLE IF NOT EXISTS result_sets ( result_set_id TEXT PRIMARY KEY NOT NULL CHECK(length(trim(result_set_id)) > 0), query_hash TEXT NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, @@ -98,6 +103,32 @@ CREATE INDEX IF NOT EXISTS idx_query_edges_dst_kind ON query_edges(dst_query_hash, edge_kind, created_at_ms DESC); +-- Durable retained run mappings are intentionally separate from disposable +-- ops telemetry: only explicitly retained relations may back `from query-run:`. +CREATE TABLE IF NOT EXISTS retained_query_runs ( + run_id TEXT PRIMARY KEY NOT NULL CHECK(run_id GLOB 'qr_*' AND length(run_id) > 3), + query_hash TEXT NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, + result_set_id TEXT NOT NULL REFERENCES result_sets(result_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT, + retained_at_ms INTEGER NOT NULL CHECK(retained_at_ms >= 0) +) STRICT; + +CREATE TABLE IF NOT EXISTS query_evaluation_receipts ( + receipt_id TEXT PRIMARY KEY NOT NULL CHECK(length(trim(receipt_id)) > 0), + query_hash TEXT NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, + result_set_id TEXT REFERENCES result_sets(result_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT, + source_generation TEXT NOT NULL CHECK(length(trim(source_generation)) > 0), + user_generation TEXT NOT NULL CHECK(length(trim(user_generation)) > 0), + index_generation TEXT NOT NULL CHECK(length(trim(index_generation)) > 0), + runtime_build_ref TEXT NOT NULL CHECK(length(trim(runtime_build_ref)) > 0), + model_refs_json TEXT NOT NULL DEFAULT '[]' CHECK(json_valid(model_refs_json) AND json_type(model_refs_json) = 'array'), + resolved_bounds_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(resolved_bounds_json) AND json_type(resolved_bounds_json) = 'object'), + degradation_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(degradation_json) AND json_type(degradation_json) = 'object'), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_query_evaluation_receipts_query_time +ON query_evaluation_receipts(query_hash, created_at_ms DESC); + -- Immutable versioned annotation construct definitions. Definition JSON is -- canonical and fingerprinted in Python; the row-level identity cannot be -- reused for a different construct after registration. diff --git a/polylogue/storage/sqlite/migration_runner.py b/polylogue/storage/sqlite/migration_runner.py index 3dca762aa9..6c7c0e7435 100644 --- a/polylogue/storage/sqlite/migration_runner.py +++ b/polylogue/storage/sqlite/migration_runner.py @@ -535,7 +535,9 @@ def migrate_archive_tier( f"{tier.value} migration {step.name} expected version {step.version - 1}, found {before}" ) _execute_migration_sql(conn, step.sql) - if tier is ArchiveTier.USER and step.version == 7: + # Saved-query migration now runs after v8 has installed the + # definition-version column consumed by the canonical identity API. + if tier is ArchiveTier.USER and step.version == 8: from polylogue.storage.sqlite.query_objects import migrate_saved_query_assertions migrate_saved_query_assertions(conn) diff --git a/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql b/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql new file mode 100644 index 0000000000..dcf07ed0cb --- /dev/null +++ b/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql @@ -0,0 +1,33 @@ +-- Versioned planner definitions and durable receipts make a query execution +-- reproducible without treating canonical identity JSON as an executable plan. +ALTER TABLE queries ADD COLUMN definition_protocol_version TEXT NOT NULL + DEFAULT 'polylogue.query-definition.v0' + CHECK(length(trim(definition_protocol_version)) > 0); + +ALTER TABLE query_names ADD COLUMN watch INTEGER NOT NULL DEFAULT 0 + CHECK(watch IN (0, 1)); +CREATE INDEX IF NOT EXISTS idx_query_names_watch +ON query_names(watch, updated_at_ms DESC, name); + +CREATE TABLE IF NOT EXISTS retained_query_runs ( + run_id TEXT PRIMARY KEY NOT NULL CHECK(run_id GLOB 'qr_*' AND length(run_id) > 3), + query_hash TEXT NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, + result_set_id TEXT NOT NULL REFERENCES result_sets(result_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT, + retained_at_ms INTEGER NOT NULL CHECK(retained_at_ms >= 0) +) STRICT; + +CREATE TABLE IF NOT EXISTS query_evaluation_receipts ( + receipt_id TEXT PRIMARY KEY NOT NULL CHECK(length(trim(receipt_id)) > 0), + query_hash TEXT NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, + result_set_id TEXT REFERENCES result_sets(result_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT, + source_generation TEXT NOT NULL CHECK(length(trim(source_generation)) > 0), + user_generation TEXT NOT NULL CHECK(length(trim(user_generation)) > 0), + index_generation TEXT NOT NULL CHECK(length(trim(index_generation)) > 0), + runtime_build_ref TEXT NOT NULL CHECK(length(trim(runtime_build_ref)) > 0), + model_refs_json TEXT NOT NULL DEFAULT '[]' CHECK(json_valid(model_refs_json) AND json_type(model_refs_json) = 'array'), + resolved_bounds_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(resolved_bounds_json) AND json_type(resolved_bounds_json) = 'object'), + degradation_json TEXT NOT NULL DEFAULT '{}' CHECK(json_valid(degradation_json) AND json_type(degradation_json) = 'object'), + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) +) STRICT; +CREATE INDEX IF NOT EXISTS idx_query_evaluation_receipts_query_time +ON query_evaluation_receipts(query_hash, created_at_ms DESC); diff --git a/polylogue/storage/sqlite/query_objects.py b/polylogue/storage/sqlite/query_objects.py index a5b748386c..d209f640e8 100644 --- a/polylogue/storage/sqlite/query_objects.py +++ b/polylogue/storage/sqlite/query_objects.py @@ -8,14 +8,20 @@ from typing import Literal from polylogue.core.hashing import hash_payload -from polylogue.core.query_identity import JsonValue, canonical_query_plan, query_hash_for_plan, query_ref +from polylogue.core.query_identity import ( + QUERY_DEFINITION_PROTOCOL_VERSION, + JsonValue, + canonical_query_plan, + query_hash_for_plan, + query_ref, +) QueryEdgeKind = Literal["operand-of", "refines", "supersedes", "derived-from", "same-as"] ResultSetExactness = Literal["exact", "capped", "sampled", "estimate"] ResultSetPersistence = Literal["routine", "watch", "pinned", "finding", "cohort"] _DURABLE_MEMBER_PERSISTENCE = frozenset({"watch", "pinned", "finding", "cohort"}) -_ACYCLIC_EDGE_KINDS = frozenset({"supersedes", "derived-from"}) +_ACYCLIC_EDGE_KINDS = frozenset({"operand-of", "supersedes", "derived-from"}) @dataclass(frozen=True, slots=True) @@ -25,6 +31,7 @@ class QueryObject: grain: str lane: str rank_policy: str + definition_protocol_version: str @property def ref(self) -> str: @@ -44,6 +51,25 @@ class ResultSetManifest: persistence_class: ResultSetPersistence +@dataclass(frozen=True, slots=True) +class RetainedQueryRun: + run_id: str + query_hash: str + result_set_id: str + + +@dataclass(frozen=True, slots=True) +class EvaluationReceipt: + receipt_id: str + source_generation: str + user_generation: str + index_generation: str + runtime_build_ref: str + model_refs: tuple[str, ...] = () + resolved_bounds: dict[str, JsonValue] | None = None + degradation: dict[str, JsonValue] | None = None + + def put_query( conn: sqlite3.Connection, planned_ast: dict[str, JsonValue], @@ -52,24 +78,37 @@ def put_query( lane: str, rank_policy: str, field_aliases: dict[str, str] | None = None, + definition_protocol_version: str = QUERY_DEFINITION_PROTOCOL_VERSION, created_at_ms: int, ) -> QueryObject: """Idempotently persist one canonical expanded query plan.""" canonical_plan = canonical_query_plan( - planned_ast, grain=grain, lane=lane, rank_policy=rank_policy, field_aliases=field_aliases + planned_ast, + grain=grain, + lane=lane, + rank_policy=rank_policy, + field_aliases=field_aliases, + definition_protocol_version=definition_protocol_version, ) query_hash = query_hash_for_plan( - planned_ast, grain=grain, lane=lane, rank_policy=rank_policy, field_aliases=field_aliases + planned_ast, + grain=grain, + lane=lane, + rank_policy=rank_policy, + field_aliases=field_aliases, + definition_protocol_version=definition_protocol_version, ) conn.execute( """ - INSERT INTO queries (query_hash, canonical_plan_json, grain, lane, rank_policy, created_at_ms) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO queries ( + query_hash, canonical_plan_json, grain, lane, rank_policy, + definition_protocol_version, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(query_hash) DO NOTHING """, - (query_hash, _json(canonical_plan), grain, lane, rank_policy, created_at_ms), + (query_hash, _json(canonical_plan), grain, lane, rank_policy, definition_protocol_version, created_at_ms), ) - return QueryObject(query_hash, canonical_plan, grain, lane, rank_policy) + return QueryObject(query_hash, canonical_plan, grain, lane, rank_policy, definition_protocol_version) def put_query_name( @@ -79,18 +118,20 @@ def put_query_name( query_hash: str, updated_at_ms: int, supersedes_query_hash: str | None = None, + watch: bool = False, ) -> None: """Move a mutable human query name to an immutable query hash.""" conn.execute( """ - INSERT INTO query_names (name, query_hash, supersedes_query_hash, updated_at_ms) - VALUES (?, ?, ?, ?) + INSERT INTO query_names (name, query_hash, supersedes_query_hash, watch, updated_at_ms) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(name) DO UPDATE SET query_hash = excluded.query_hash, supersedes_query_hash = excluded.supersedes_query_hash, + watch = excluded.watch, updated_at_ms = excluded.updated_at_ms """, - (name, query_hash, supersedes_query_hash, updated_at_ms), + (name, query_hash, supersedes_query_hash, int(watch), updated_at_ms), ) @@ -117,7 +158,7 @@ def put_result_set( grain=grain, corpus_epoch=corpus_epoch, member_count=len(member_refs), - membership_merkle_root=_membership_merkle_root(member_refs), + membership_merkle_root=membership_merkle_root(member_refs), ordered_rank_hash=hash_payload(list(member_refs)), exactness=exactness, persistence_class=persistence_class, @@ -150,6 +191,158 @@ def put_result_set( return manifest +def get_query(conn: sqlite3.Connection, query_hash: str) -> QueryObject | None: + """Read one immutable query definition without decoding it as executable syntax.""" + row = conn.execute( + """ + SELECT query_hash, canonical_plan_json, grain, lane, rank_policy, definition_protocol_version + FROM queries WHERE query_hash = ? + """, + (query_hash,), + ).fetchone() + if row is None: + return None + payload = json.loads(str(row[1])) + if not isinstance(payload, dict): + raise ValueError(f"query {query_hash} has invalid canonical plan") + return QueryObject( + query_hash=str(row[0]), + canonical_plan=payload, + grain=str(row[2]), + lane=str(row[3]), + rank_policy=str(row[4]), + definition_protocol_version=str(row[5]), + ) + + +def list_watched_queries(conn: sqlite3.Connection) -> tuple[QueryObject, ...]: + """Return current mutable-name watch definitions, one name per query version.""" + rows = conn.execute( + """ + SELECT q.query_hash, q.canonical_plan_json, q.grain, q.lane, q.rank_policy, + q.definition_protocol_version + FROM query_names AS n + JOIN queries AS q ON q.query_hash = n.query_hash + WHERE n.watch = 1 + ORDER BY n.updated_at_ms, n.name + """ + ).fetchall() + return tuple( + QueryObject( + query_hash=str(row[0]), + canonical_plan=_query_payload(row[1]), + grain=str(row[2]), + lane=str(row[3]), + rank_policy=str(row[4]), + definition_protocol_version=str(row[5]), + ) + for row in rows + ) + + +def get_result_set(conn: sqlite3.Connection, result_set_id: str) -> ResultSetManifest | None: + row = conn.execute( + """ + SELECT result_set_id, query_hash, grain, corpus_epoch, member_count, + membership_merkle_root, ordered_rank_hash, exactness, persistence_class + FROM result_sets WHERE result_set_id = ? + """, + (result_set_id,), + ).fetchone() + return _manifest_from_row(row) if row is not None else None + + +def get_result_set_members(conn: sqlite3.Connection, result_set_id: str) -> tuple[str, ...]: + rows = conn.execute( + "SELECT member_ref FROM result_set_members WHERE result_set_id = ? ORDER BY rank", + (result_set_id,), + ).fetchall() + return tuple(str(row[0]) for row in rows) + + +def get_latest_result_set( + conn: sqlite3.Connection, + *, + query_hash: str, + persistence_class: ResultSetPersistence, +) -> ResultSetManifest | None: + row = conn.execute( + """ + SELECT result_set_id, query_hash, grain, corpus_epoch, member_count, + membership_merkle_root, ordered_rank_hash, exactness, persistence_class + FROM result_sets + WHERE query_hash = ? AND persistence_class = ? + ORDER BY created_at_ms DESC, result_set_id DESC + LIMIT 1 + """, + (query_hash, persistence_class), + ).fetchone() + return _manifest_from_row(row) if row is not None else None + + +def put_retained_query_run( + conn: sqlite3.Connection, + *, + run_id: str, + query_hash: str, + result_set_id: str, + retained_at_ms: int, +) -> RetainedQueryRun: + conn.execute( + """ + INSERT INTO retained_query_runs (run_id, query_hash, result_set_id, retained_at_ms) + VALUES (?, ?, ?, ?) + ON CONFLICT(run_id) DO UPDATE SET + query_hash = excluded.query_hash, + result_set_id = excluded.result_set_id, + retained_at_ms = excluded.retained_at_ms + """, + (run_id, query_hash, result_set_id, retained_at_ms), + ) + return RetainedQueryRun(run_id, query_hash, result_set_id) + + +def get_retained_query_run(conn: sqlite3.Connection, run_id: str) -> RetainedQueryRun | None: + row = conn.execute( + "SELECT run_id, query_hash, result_set_id FROM retained_query_runs WHERE run_id = ?", (run_id,) + ).fetchone() + return RetainedQueryRun(str(row[0]), str(row[1]), str(row[2])) if row is not None else None + + +def put_evaluation_receipt( + conn: sqlite3.Connection, + *, + query_hash: str, + receipt: EvaluationReceipt, + result_set_id: str | None, + created_at_ms: int, +) -> None: + """Persist immutable execution context for a materialized relation.""" + conn.execute( + """ + INSERT INTO query_evaluation_receipts ( + receipt_id, query_hash, result_set_id, source_generation, user_generation, + index_generation, runtime_build_ref, model_refs_json, resolved_bounds_json, + degradation_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(receipt_id) DO NOTHING + """, + ( + receipt.receipt_id, + query_hash, + result_set_id, + receipt.source_generation, + receipt.user_generation, + receipt.index_generation, + receipt.runtime_build_ref, + _json(list(receipt.model_refs)), + _json(receipt.resolved_bounds or {}), + _json(receipt.degradation or {}), + created_at_ms, + ), + ) + + def put_query_edge( conn: sqlite3.Connection, *, @@ -216,7 +409,7 @@ def migrate_saved_query_assertions(conn: sqlite3.Connection) -> int: return len(rows) -def _membership_merkle_root(member_refs: tuple[str, ...]) -> str: +def membership_merkle_root(member_refs: tuple[str, ...]) -> str: if not member_refs: return hash_payload([]) level = sorted(hash_payload(member_ref) for member_ref in member_refs) @@ -231,12 +424,47 @@ def _json(value: object) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) +def _query_payload(value: object) -> dict[str, JsonValue]: + parsed = json.loads(str(value)) + if not isinstance(parsed, dict): + raise ValueError("query canonical plan must be an object") + return parsed + + +def _manifest_from_row(row: sqlite3.Row | tuple[object, ...]) -> ResultSetManifest: + member_count = row[4] + if isinstance(member_count, bool) or not isinstance(member_count, int): + raise ValueError("result set member_count must be an integer") + return ResultSetManifest( + result_set_id=str(row[0]), + query_hash=str(row[1]), + grain=str(row[2]), + corpus_epoch=str(row[3]), + member_count=member_count, + membership_merkle_root=str(row[5]), + ordered_rank_hash=str(row[6]), + exactness=str(row[7]), # type: ignore[arg-type] + persistence_class=str(row[8]), # type: ignore[arg-type] + ) + + __all__ = [ + "EvaluationReceipt", "QueryObject", + "RetainedQueryRun", "ResultSetManifest", + "get_latest_result_set", + "get_query", + "get_result_set", + "get_result_set_members", + "get_retained_query_run", + "list_watched_queries", + "membership_merkle_root", "migrate_saved_query_assertions", + "put_evaluation_receipt", "put_query", "put_query_edge", "put_query_name", + "put_retained_query_run", "put_result_set", ] diff --git a/tests/unit/archive/query/test_evaluator.py b/tests/unit/archive/query/test_evaluator.py new file mode 100644 index 0000000000..cfb52a7ee4 --- /dev/null +++ b/tests/unit/archive/query/test_evaluator.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import sqlite3 + +import pytest + +from polylogue.archive.query.evaluator import ( + DurableRefResolver, + QueryEvaluation, + QueryEvaluationRequest, + RetainedRelationUnavailableError, +) +from polylogue.archive.query.expression import RefOperand, resolve_ref_operand +from polylogue.core.refs import ObjectRef +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.query_objects import EvaluationReceipt, put_query, put_result_set, put_retained_query_run + + +class _Evaluator: + def __init__(self) -> None: + self.requests: list[QueryEvaluationRequest] = [] + + def evaluate(self, request: QueryEvaluationRequest) -> QueryEvaluation: + self.requests.append(request) + return QueryEvaluation( + grain="session", + member_refs=("session:re-evaluated",), + corpus_epoch="index:g1", + exactness="exact", + receipt=EvaluationReceipt("receipt", "source:g1", "user:g1", "index:g1", "build:test"), + ) + + def resolve_cohort(self, operand: RefOperand) -> QueryEvaluation: + assert operand.reference.format() == "cohort:team" + return QueryEvaluation( + grain="session", + member_refs=("session:cohort",), + corpus_epoch="index:g1", + exactness="exact", + receipt=EvaluationReceipt("cohort-receipt", "source:g1", "user:g1", "index:g1", "build:test"), + ) + + +def _conn() -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.execute("PRAGMA foreign_keys = ON") + initialize_archive_tier(conn, ArchiveTier.USER) + return conn + + +def test_resolver_uses_planner_evaluation_and_retained_relations() -> None: + conn = _conn() + parent = put_query( + conn, + {"field": "title", "value": "parent"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + child = put_query( + conn, + {"field": "title", "value": "child"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + result = put_result_set( + conn, + result_set_id="retained", + query_hash=child.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:stored",), + exactness="exact", + persistence_class="pinned", + created_at_ms=2, + ) + put_retained_query_run( + conn, run_id="qr_saved", query_hash=child.query_hash, result_set_id=result.result_set_id, retained_at_ms=3 + ) + evaluator = _Evaluator() + resolver = DurableRefResolver(conn, evaluator, owner_query_hash=parent.query_hash, created_at_ms=4) + + dynamic = resolve_ref_operand(RefOperand(ObjectRef(kind="query", object_id=child.query_hash)), resolver) + retained = resolve_ref_operand(RefOperand(ObjectRef(kind="query-run", object_id="qr_saved")), resolver) + cohort = resolve_ref_operand(RefOperand(ObjectRef(kind="cohort", object_id="team")), resolver) + + assert dynamic.member_refs == ("session:re-evaluated",) + assert evaluator.requests[0].query == child + assert retained.member_refs == ("session:stored",) + assert cohort.member_refs == ("session:cohort",) + assert conn.execute("SELECT edge_kind FROM query_edges").fetchone()[0] == "operand-of" + + +def test_query_run_without_retained_relation_fails_closed() -> None: + with pytest.raises(RetainedRelationUnavailableError, match="no retained relation"): + resolve_ref_operand( + RefOperand(ObjectRef(kind="query-run", object_id="qr_missing")), + DurableRefResolver(_conn(), _Evaluator()), + ) diff --git a/tests/unit/core/test_query_identity.py b/tests/unit/core/test_query_identity.py index 9af1ad7781..a5600b3bbd 100644 --- a/tests/unit/core/test_query_identity.py +++ b/tests/unit/core/test_query_identity.py @@ -69,3 +69,21 @@ def test_canonical_query_plan_retains_relative_time_as_dynamic_ast() -> None: dynamic: dict[str, JsonValue] = {"field": "since", "relative": "7d"} assert canonical_query_plan(dynamic, grain="session", lane="dialogue", rank_policy="mixed")["ast"] == dynamic + + +def test_definition_protocol_version_is_bound_into_query_identity() -> None: + plan: dict[str, JsonValue] = {"field": "origin", "value": "codex-session"} + + assert query_hash_for_plan( + plan, + grain="session", + lane="dialogue", + rank_policy="mixed", + definition_protocol_version="polylogue.query-definition.v1", + ) != query_hash_for_plan( + plan, + grain="session", + lane="dialogue", + rank_policy="mixed", + definition_protocol_version="polylogue.query-definition.v2", + ) diff --git a/tests/unit/daemon/test_convergence_stages.py b/tests/unit/daemon/test_convergence_stages.py index 342c613425..2f51056d5e 100644 --- a/tests/unit/daemon/test_convergence_stages.py +++ b/tests/unit/daemon/test_convergence_stages.py @@ -850,7 +850,7 @@ def test_default_convergence_stages_always_register_embed_stage( stage_names = [stage.name for stage in make_default_convergence_stages(tmp_path / "archive.sqlite")] - assert stage_names == ["fts", "embed", "insights"] + assert stage_names == ["fts", "embed", "insights", "standing-queries"] def test_embed_stage_is_noop_when_disabled( diff --git a/tests/unit/daemon/test_standing_queries.py b/tests/unit/daemon/test_standing_queries.py new file mode 100644 index 0000000000..72b9b83b85 --- /dev/null +++ b/tests/unit/daemon/test_standing_queries.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from polylogue.archive.query.evaluator import QueryEvaluation, QueryEvaluationRequest +from polylogue.core.enums import AssertionKind, AssertionStatus +from polylogue.daemon.convergence_stages import make_standing_query_stage +from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.archive_tiers.user_write import ( + FindingAssertion, + mark_assertion_status, + upsert_findings_as_assertions, +) +from polylogue.storage.sqlite.query_objects import EvaluationReceipt, put_query, put_query_name + + +class _Evaluator: + def __init__(self, *, members: tuple[str, ...], cache_only: bool = False) -> None: + self.members = members + self.cache_only = cache_only + self.requests: list[QueryEvaluationRequest] = [] + + def evaluate(self, request: QueryEvaluationRequest) -> QueryEvaluation: + self.requests.append(request) + return QueryEvaluation( + grain="session", + member_refs=self.members, + corpus_epoch="index:g1", + exactness="exact", + cache_only=self.cache_only, + receipt=EvaluationReceipt( + receipt_id=f"receipt-{len(self.requests)}-{request.purpose}", + source_generation="source:g1", + user_generation="user:g1", + index_generation="index:g1", + runtime_build_ref="build:test", + ), + ) + + def resolve_cohort(self, _operand: object) -> QueryEvaluation: + raise AssertionError("standing queries do not resolve cohorts directly") + + +def _seed_watch(tmp_path: Path) -> tuple[Path, str]: + index_db = tmp_path / "index.db" + user_db = tmp_path / "user.db" + initialize_archive_database(user_db, ArchiveTier.USER) + with sqlite3.connect(user_db) as conn: + query = put_query( + conn, + {"field": "origin", "value": "codex-session"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + put_query_name(conn, name="codex", query_hash=query.query_hash, watch=True, updated_at_ms=2) + conn.commit() + return index_db, query.query_hash + + +def test_watched_session_delta_is_candidate_once_and_has_self_firewall(tmp_path: Path) -> None: + index_db, query_hash = _seed_watch(tmp_path) + evaluator = _Evaluator(members=("session:one",)) + stage = make_standing_query_stage(index_db, evaluator=evaluator) + + assert stage.check_sessions is not None and stage.execute_sessions is not None + assert stage.check_sessions(("session:changed",)) == {"session:changed"} + assert stage.execute_sessions(("session:changed",)) is True # baseline only + evaluator.members = ("session:one", "session:two") + assert stage.execute_sessions(("session:changed",)) is True + assert stage.execute_sessions(("session:changed",)) is True + + with sqlite3.connect(tmp_path / "user.db") as conn: + rows = conn.execute("SELECT status, target_ref, value_json FROM assertions WHERE kind = 'finding'").fetchall() + assert len(rows) == 1 + assert rows[0][0] == "candidate" + assert rows[0][1] == f"query:{query_hash}" + assert "query-delta" in str(rows[0][2]) + assert conn.execute("SELECT COUNT(*) FROM query_evaluation_receipts").fetchone()[0] == 3 + request = evaluator.requests[0] + assert request.excluded_scope_refs == (f"query:{query_hash}",) + assert request.excluded_origin_prefixes == ("notice.",) + + +def test_cache_only_evaluation_after_index_reset_never_compares_baseline(tmp_path: Path) -> None: + index_db, _ = _seed_watch(tmp_path) + evaluator = _Evaluator(members=("session:one",)) + stage = make_standing_query_stage(index_db, evaluator=evaluator) + assert stage.execute_sessions is not None + assert stage.execute_sessions(("session:changed",)) is True + evaluator.members = ("session:one", "session:two") + evaluator.cache_only = True + assert stage.execute_sessions(("session:changed",)) is True + + with sqlite3.connect(tmp_path / "user.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM assertions WHERE kind = 'finding'").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM result_sets WHERE persistence_class = 'watch'").fetchone()[0] == 1 + + +def test_promoted_expected_count_divergence_targets_original_finding(tmp_path: Path) -> None: + index_db, query_hash = _seed_watch(tmp_path) + with sqlite3.connect(tmp_path / "user.db") as conn: + original = upsert_findings_as_assertions( + conn, + [ + FindingAssertion( + claim_key="expected-count", + target_ref=f"query:{query_hash}", + body_text="Expected one member.", + finding_kind="measure", + statistic={"op": "count", "value": 1, "unit": "members"}, + n=1, + query_ref=f"query:{query_hash}", + result_set_ref="result-set:original", + detector_ref="agent:test-detector", + expected={"measure": "member_count", "op": "=", "value": 1}, + ) + ], + now_ms=1, + )[0] + mark_assertion_status(conn, original.assertion_id, AssertionStatus.ACCEPTED, now_ms=2) + conn.commit() + stage = make_standing_query_stage(index_db, evaluator=_Evaluator(members=("session:one", "session:two"))) + assert stage.execute_sessions is not None + assert stage.execute_sessions(("session:changed",)) is True + + with sqlite3.connect(tmp_path / "user.db") as conn: + rows = conn.execute( + "SELECT target_ref, value_json, status FROM assertions WHERE kind = ? ORDER BY created_at_ms, assertion_id", + (AssertionKind.FINDING.value,), + ).fetchall() + assert len(rows) == 2 + assert rows[1][0] == f"assertion:{original.assertion_id}" + assert '"finding_kind":"query-drift"' in str(rows[1][1]) + assert rows[1][2] == "candidate" diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index 24fbde48eb..9a34539ac0 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -265,7 +265,7 @@ def test_user_tier_v3_migrates_to_current_with_verified_backup_receipt( result = migrate_archive_tier(conn, ArchiveTier.USER, backup_manifest=manifest) assert result.from_version == 3 assert result.to_version == USER_SCHEMA_VERSION - assert result.applied_versions == (4, 5, 6, 7) + assert result.applied_versions == (4, 5, 6, 7, 8) assert result.backup_receipt == manifest.with_name("verification-receipt.json") assert int(conn.execute("PRAGMA user_version").fetchone()[0]) == USER_SCHEMA_VERSION assert conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='user_settings'").fetchone() @@ -319,8 +319,8 @@ def test_user_tier_v5_annotation_migration_requires_verified_backup_and_matches_ try: result = migrate_archive_tier(conn, ArchiveTier.USER, backup_manifest=manifest) assert result.from_version == 5 - assert result.to_version == USER_SCHEMA_VERSION == 7 - assert result.applied_versions == (6, 7) + assert result.to_version == USER_SCHEMA_VERSION == 8 + assert result.applied_versions == (6, 7, 8) assert result.backup_receipt == manifest.with_name("verification-receipt.json") assert conn.execute("SELECT assertion_id FROM assertions WHERE assertion_id = 'sentinel'").fetchone() saved_target = conn.execute( diff --git a/tests/unit/storage/test_query_objects.py b/tests/unit/storage/test_query_objects.py index 300e3c6ade..7737ed89ef 100644 --- a/tests/unit/storage/test_query_objects.py +++ b/tests/unit/storage/test_query_objects.py @@ -8,10 +8,17 @@ from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.query_objects import ( + EvaluationReceipt, + get_query, + get_retained_query_run, + list_watched_queries, migrate_saved_query_assertions, + put_evaluation_receipt, put_query, put_query_edge, + put_query_name, put_result_set, + put_retained_query_run, ) @@ -113,3 +120,56 @@ def test_saved_query_migration_preserves_all_assertions_and_repoints_targets() - target_ref = conn.execute("SELECT target_ref FROM assertions WHERE assertion_id = 'saved'").fetchone()[0] assert str(target_ref).startswith("query:") assert conn.execute("SELECT COUNT(*) FROM queries").fetchone()[0] == 1 + + +def test_watched_definition_retention_and_receipt_are_durable_contracts() -> None: + conn = _conn() + query = put_query( + conn, + {"field": "origin", "value": "codex-session"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + put_query_name(conn, name="codex", query_hash=query.query_hash, watch=True, updated_at_ms=2) + result = put_result_set( + conn, + result_set_id="watch-result", + query_hash=query.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:one",), + exactness="exact", + persistence_class="watch", + created_at_ms=3, + ) + put_retained_query_run( + conn, + run_id="qr_saved", + query_hash=query.query_hash, + result_set_id=result.result_set_id, + retained_at_ms=4, + ) + put_evaluation_receipt( + conn, + query_hash=query.query_hash, + result_set_id=result.result_set_id, + receipt=EvaluationReceipt( + receipt_id="receipt-1", + source_generation="source:g1", + user_generation="user:g1", + index_generation="index:g1", + runtime_build_ref="build:test", + model_refs=("model:none",), + ), + created_at_ms=5, + ) + + restored = get_query(conn, query.query_hash) + assert restored is not None + assert restored.definition_protocol_version == "polylogue.query-definition.v1" + assert restored.canonical_plan["definition_protocol_version"] == "polylogue.query-definition.v1" + assert list_watched_queries(conn) == (restored,) + assert get_retained_query_run(conn, "qr_saved") is not None + assert conn.execute("SELECT result_set_id FROM query_evaluation_receipts").fetchone()[0] == result.result_set_id From 0b734590b302e641821bb0bd65ae72ba99e86295 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 07:23:30 +0200 Subject: [PATCH 04/10] fix(query): schedule retained query findings Problem: accepted expected-count findings were skipped without a watched name, and retained run rows could point to another query's result set. What changed: schedule promoted expected findings through the existing session convergence hook even without a watch, and enforce query/result set agreement when retaining or resolving a run. Ref polylogue-rxdo.5 Ref polylogue-rxdo.6 Co-Authored-By: Claude --- polylogue/archive/query/evaluator.py | 6 ++++ polylogue/daemon/convergence_stages.py | 17 ++++++++- polylogue/storage/sqlite/query_objects.py | 5 +++ tests/unit/archive/query/test_evaluator.py | 40 ++++++++++++++++++++++ tests/unit/daemon/test_standing_queries.py | 15 ++++---- 5 files changed, 76 insertions(+), 7 deletions(-) diff --git a/polylogue/archive/query/evaluator.py b/polylogue/archive/query/evaluator.py index b0125e0165..482ede940e 100644 --- a/polylogue/archive/query/evaluator.py +++ b/polylogue/archive/query/evaluator.py @@ -117,6 +117,12 @@ def _retained_result( manifest = get_result_set(self._conn, result_set_id) if manifest is None: raise RetainedRelationUnavailableError(f"retained result-set:{result_set_id} is unavailable") + if extra_lineage: + run = get_retained_query_run(self._conn, extra_lineage[0].object_id) + if run is None or manifest.query_hash != run.query_hash: + raise RetainedRelationUnavailableError( + f"query run {extra_lineage[0].format()} does not retain a relation for its query" + ) members = get_result_set_members(self._conn, result_set_id) if manifest.member_count != len(members): raise RetainedRelationUnavailableError( diff --git a/polylogue/daemon/convergence_stages.py b/polylogue/daemon/convergence_stages.py index 41f8281edc..386be73998 100644 --- a/polylogue/daemon/convergence_stages.py +++ b/polylogue/daemon/convergence_stages.py @@ -601,7 +601,9 @@ def check_sessions(session_ids: Sequence[str]) -> set[str]: return set() try: with sqlite3.connect(f"file:{user_db}?mode=ro", uri=True, timeout=5.0) as conn: - return set(session_ids) if list_watched_queries(conn) else set() + if list_watched_queries(conn) or _has_promoted_expected_findings(conn): + return set(session_ids) + return set() except Exception: logger.warning("standing-queries: watch lookup failed", exc_info=True) return set(session_ids) @@ -806,6 +808,19 @@ def _materialize_promoted_finding_drifts( ) +def _has_promoted_expected_findings(conn: sqlite3.Connection) -> bool: + """Return whether convergence must run expected findings without a watch.""" + for finding in list_assertion_claims( + conn, + kinds=(AssertionKind.FINDING,), + statuses=(AssertionStatus.ACCEPTED,), + ): + value = finding.value if isinstance(finding.value, dict) else {} + if isinstance(value.get("expected"), dict): + return True + return False + + def _matches_expected_count(expected: Mapping[str, object], actual: int) -> bool: if expected.get("measure") != "member_count": return True diff --git a/polylogue/storage/sqlite/query_objects.py b/polylogue/storage/sqlite/query_objects.py index d209f640e8..c53d184826 100644 --- a/polylogue/storage/sqlite/query_objects.py +++ b/polylogue/storage/sqlite/query_objects.py @@ -288,6 +288,11 @@ def put_retained_query_run( result_set_id: str, retained_at_ms: int, ) -> RetainedQueryRun: + result_set = get_result_set(conn, result_set_id) + if result_set is None: + raise KeyError(f"result-set:{result_set_id}") + if result_set.query_hash != query_hash: + raise ValueError("retained query run result set must belong to the same query") conn.execute( """ INSERT INTO retained_query_runs (run_id, query_hash, result_set_id, retained_at_ms) diff --git a/tests/unit/archive/query/test_evaluator.py b/tests/unit/archive/query/test_evaluator.py index cfb52a7ee4..20beca6fe9 100644 --- a/tests/unit/archive/query/test_evaluator.py +++ b/tests/unit/archive/query/test_evaluator.py @@ -101,3 +101,43 @@ def test_query_run_without_retained_relation_fails_closed() -> None: RefOperand(ObjectRef(kind="query-run", object_id="qr_missing")), DurableRefResolver(_conn(), _Evaluator()), ) + + +def test_retained_query_run_rejects_a_result_set_for_another_query() -> None: + conn = _conn() + first = put_query( + conn, + {"field": "title", "value": "first"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + second = put_query( + conn, + {"field": "title", "value": "second"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + result = put_result_set( + conn, + result_set_id="second-result", + query_hash=second.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:second",), + exactness="exact", + persistence_class="pinned", + created_at_ms=2, + ) + + with pytest.raises(ValueError, match="same query"): + put_retained_query_run( + conn, + run_id="qr_mismatch", + query_hash=first.query_hash, + result_set_id=result.result_set_id, + retained_at_ms=3, + ) diff --git a/tests/unit/daemon/test_standing_queries.py b/tests/unit/daemon/test_standing_queries.py index 72b9b83b85..588e6d43a8 100644 --- a/tests/unit/daemon/test_standing_queries.py +++ b/tests/unit/daemon/test_standing_queries.py @@ -5,6 +5,7 @@ from polylogue.archive.query.evaluator import QueryEvaluation, QueryEvaluationRequest from polylogue.core.enums import AssertionKind, AssertionStatus +from polylogue.daemon.convergence import DaemonConverger from polylogue.daemon.convergence_stages import make_standing_query_stage from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier @@ -43,7 +44,7 @@ def resolve_cohort(self, _operand: object) -> QueryEvaluation: raise AssertionError("standing queries do not resolve cohorts directly") -def _seed_watch(tmp_path: Path) -> tuple[Path, str]: +def _seed_watch(tmp_path: Path, *, watch: bool = True) -> tuple[Path, str]: index_db = tmp_path / "index.db" user_db = tmp_path / "user.db" initialize_archive_database(user_db, ArchiveTier.USER) @@ -56,7 +57,8 @@ def _seed_watch(tmp_path: Path) -> tuple[Path, str]: rank_policy="mixed", created_at_ms=1, ) - put_query_name(conn, name="codex", query_hash=query.query_hash, watch=True, updated_at_ms=2) + if watch: + put_query_name(conn, name="codex", query_hash=query.query_hash, watch=True, updated_at_ms=2) conn.commit() return index_db, query.query_hash @@ -100,8 +102,8 @@ def test_cache_only_evaluation_after_index_reset_never_compares_baseline(tmp_pat assert conn.execute("SELECT COUNT(*) FROM result_sets WHERE persistence_class = 'watch'").fetchone()[0] == 1 -def test_promoted_expected_count_divergence_targets_original_finding(tmp_path: Path) -> None: - index_db, query_hash = _seed_watch(tmp_path) +def test_promoted_expected_count_divergence_targets_original_finding_without_watch(tmp_path: Path) -> None: + index_db, query_hash = _seed_watch(tmp_path, watch=False) with sqlite3.connect(tmp_path / "user.db") as conn: original = upsert_findings_as_assertions( conn, @@ -124,8 +126,9 @@ def test_promoted_expected_count_divergence_targets_original_finding(tmp_path: P mark_assertion_status(conn, original.assertion_id, AssertionStatus.ACCEPTED, now_ms=2) conn.commit() stage = make_standing_query_stage(index_db, evaluator=_Evaluator(members=("session:one", "session:two"))) - assert stage.execute_sessions is not None - assert stage.execute_sessions(("session:changed",)) is True + converger = DaemonConverger(stages=(stage,), max_workers=1) + states, _ = converger.converge_sessions(("session:changed",)) + assert states["session:changed"].stages["standing-queries"].value == "done" with sqlite3.connect(tmp_path / "user.db") as conn: rows = conn.execute( From 87de23781b25145b791c3ba540e8dcef4354842e Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 07:36:20 +0200 Subject: [PATCH 05/10] refactor(daemon): isolate standing query stage Problem: the convergence-stage retro requires extracting a stage before\nadding a fourth one, and multiple watched aliases could evaluate the same\nimmutable definition twice in a single convergence pass.\n\nWhat changed: move standing-query materialization into its own daemon\nmodule while retaining the existing factory export, and deduplicate watched\nquery versions at the durable lookup boundary. Add a regression covering\ntwo aliases resolving to one evaluation and no fabricated initial delta.\n\nRef polylogue-rxdo.5\nRef polylogue-rxdo.6\n\nCo-Authored-By: Claude --- docs/plans/topology-target.yaml | 10 +- docs/topology-status.md | 6 +- polylogue/daemon/convergence_stages.py | 291 +---------------- .../daemon/convergence_standing_queries.py | 302 ++++++++++++++++++ polylogue/storage/sqlite/query_objects.py | 8 +- tests/unit/daemon/test_standing_queries.py | 17 + 6 files changed, 335 insertions(+), 299 deletions(-) create mode 100644 polylogue/daemon/convergence_standing_queries.py diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index 735088b6fd..f3f000e1bc 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -332,7 +332,7 @@ files: owner: archive-query reason: archive-domain query semantics - path: polylogue/archive/query/evaluator.py - loc: 161 + loc: 165 target: polylogue/archive/query/evaluator.py owner: archive-query reason: archive-domain query semantics @@ -1320,9 +1320,13 @@ files: target: polylogue/daemon/convergence_debt_status.py owner: stable - path: polylogue/daemon/convergence_stages.py - loc: 1942 + loc: 1677 target: polylogue/daemon/convergence_stages.py owner: stable + - path: polylogue/daemon/convergence_standing_queries.py + loc: 302 + target: polylogue/daemon/convergence_standing_queries.py + owner: stable - path: polylogue/daemon/cursor_lag_alert.py loc: 409 target: polylogue/daemon/cursor_lag_alert.py @@ -3774,7 +3778,7 @@ files: target: polylogue/storage/sqlite/queries/tool_usage.py owner: stable - path: polylogue/storage/sqlite/query_objects.py - loc: 467 + loc: 475 target: polylogue/storage/sqlite/query_objects.py owner: stable - path: polylogue/storage/sqlite/query_store.py diff --git a/docs/topology-status.md b/docs/topology-status.md index 27fd1c5fed..82a637c107 100644 --- a/docs/topology-status.md +++ b/docs/topology-status.md @@ -28,12 +28,12 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe ### Summary -- **Stable** (no move scoped): 774 +- **Stable** (no move scoped): 775 - **Kernel** (polylogue/ root): 10 - **Primitives** (storage-root): 18 - **TBD** (cell needs explicit assignment): 6 -- **Total declared**: 928 -- **Realized polylogue/**/*.py**: 928 files declared +- **Total declared**: 929 +- **Realized polylogue/**/*.py**: 929 files declared ### TBD cells (require explicit routing) diff --git a/polylogue/daemon/convergence_stages.py b/polylogue/daemon/convergence_stages.py index 386be73998..6fa1531d74 100644 --- a/polylogue/daemon/convergence_stages.py +++ b/polylogue/daemon/convergence_stages.py @@ -15,38 +15,21 @@ import sqlite3 import time -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING -from polylogue.archive.query.evaluator import CanonicalPlanEvaluator, QueryEvaluation, QueryEvaluationRequest from polylogue.config import load_polylogue_config -from polylogue.core.enums import AssertionKind, AssertionStatus -from polylogue.core.hashing import hash_payload -from polylogue.core.query_identity import query_ref, result_set_ref from polylogue.daemon.convergence import ConvergenceStage, StageExecuteReturn, StageExecutionResult +from polylogue.daemon.convergence_standing_queries import make_standing_query_stage from polylogue.logging import get_logger from polylogue.storage.runtime import SESSION_INSIGHT_MATERIALIZER_VERSION from polylogue.storage.source_sessions import ( session_ids_for_source_path, session_ids_for_source_paths, ) -from polylogue.storage.sqlite.archive_tiers.user_write import ( - FindingAssertion, - list_assertion_claims, - upsert_findings_as_assertions, -) from polylogue.storage.sqlite.connection_profile import open_daemon_connection -from polylogue.storage.sqlite.query_objects import ( - get_latest_result_set, - get_query, - get_result_set, - list_watched_queries, - membership_merkle_root, - put_evaluation_receipt, - put_result_set, -) if TYPE_CHECKING: pass @@ -571,276 +554,6 @@ def execute_sessions(session_ids: Sequence[str]) -> StageExecuteReturn: ) -# ── Stage: standing queries ─────────────────────────────────────── - - -def make_standing_query_stage( - db_path: Path, - *, - evaluator: CanonicalPlanEvaluator | None = None, -) -> ConvergenceStage: - """Re-evaluate watched definitions after affected session convergence. - - The canonical-plan evaluator is deliberately injected. Identity JSON is - not an executable plan, so the stage stays inert until the owning planner - is supplied by broad runtime wiring. Its durable baseline, receipt, and - candidate-finding semantics are nevertheless the production path. - """ - - def check(_path: Path) -> bool: - return False - - def execute(_path: Path) -> StageExecuteReturn: - return True - - def check_sessions(session_ids: Sequence[str]) -> set[str]: - if evaluator is None or not session_ids: - return set() - user_db = _standing_user_db_path(db_path) - if not user_db.exists(): - return set() - try: - with sqlite3.connect(f"file:{user_db}?mode=ro", uri=True, timeout=5.0) as conn: - if list_watched_queries(conn) or _has_promoted_expected_findings(conn): - return set(session_ids) - return set() - except Exception: - logger.warning("standing-queries: watch lookup failed", exc_info=True) - return set(session_ids) - - def execute_sessions(session_ids: Sequence[str]) -> StageExecuteReturn: - if evaluator is None: - return True - user_db = _standing_user_db_path(db_path) - if not user_db.exists(): - return True - now_ms = int(time.time() * 1000) - ids = tuple(dict.fromkeys(str(session_id) for session_id in session_ids if session_id)) - try: - conn = open_daemon_connection(user_db, timeout=30.0) - try: - watched = list_watched_queries(conn) - for query in watched: - query_reference = query_ref(query.query_hash).format() - evaluation = evaluator.evaluate( - QueryEvaluationRequest( - query=query, - purpose="standing-watch", - changed_session_ids=ids, - excluded_scope_refs=(query_reference,), - excluded_origin_prefixes=("notice.",), - ) - ) - if evaluation.cache_only: - # An index-only relation after reset is not evidence of - # membership drift. Compare durable user-tier baselines only. - continue - _materialize_watch_evaluation(conn, query.query_hash, evaluation, now_ms=now_ms) - _materialize_promoted_finding_drifts(conn, evaluator, now_ms=now_ms) - conn.commit() - finally: - conn.close() - return True - except Exception: - logger.warning("standing-queries: evaluation deferred", exc_info=True) - return False - - return ConvergenceStage( - name="standing-queries", - description="Re-evaluate watched query definitions and emit candidate deltas", - check=check, - execute=execute, - check_sessions=check_sessions, - execute_sessions=execute_sessions, - cpu_bound=False, - false_means_pending=True, - ) - - -def _standing_user_db_path(db_path: Path) -> Path: - archive_db = _active_archive_index_path(db_path) - base = archive_db if archive_db is not None else db_path - return base.with_name("user.db") - - -def _watch_result_set_id(query_hash: str, member_refs: tuple[str, ...]) -> str: - return f"watch-{hash_payload((query_hash, membership_merkle_root(member_refs)))}" - - -def _materialize_watch_evaluation( - conn: sqlite3.Connection, - query_hash: str, - evaluation: QueryEvaluation, - *, - now_ms: int, -) -> None: - baseline = get_latest_result_set(conn, query_hash=query_hash, persistence_class="watch") - root = membership_merkle_root(evaluation.member_refs) - if baseline is not None and baseline.membership_merkle_root == root: - put_evaluation_receipt( - conn, - query_hash=query_hash, - receipt=evaluation.receipt, - result_set_id=baseline.result_set_id, - created_at_ms=now_ms, - ) - return - result_set_id = _watch_result_set_id(query_hash, evaluation.member_refs) - current = get_result_set(conn, result_set_id) - if current is None: - current = put_result_set( - conn, - result_set_id=result_set_id, - query_hash=query_hash, - grain=evaluation.grain, - corpus_epoch=evaluation.corpus_epoch, - member_refs=evaluation.member_refs, - exactness=evaluation.exactness, - persistence_class="watch", - created_at_ms=now_ms, - ) - put_evaluation_receipt( - conn, - query_hash=query_hash, - receipt=evaluation.receipt, - result_set_id=current.result_set_id, - created_at_ms=now_ms, - ) - if baseline is None: - return - query_reference = query_ref(query_hash).format() - current_reference = result_set_ref(current.result_set_id).format() - baseline_reference = result_set_ref(baseline.result_set_id).format() - upsert_findings_as_assertions( - conn, - [ - FindingAssertion( - claim_key="standing-query-membership-delta", - target_ref=query_reference, - body_text="Watched query membership changed after archive convergence.", - finding_kind="query-delta", - statistic={"op": "count", "value": current.member_count, "unit": "members"}, - n=current.member_count, - query_ref=query_reference, - result_set_ref=current_reference, - baseline_ref=baseline_reference, - current_ref=current_reference, - detector_ref="agent:standing-query-detector.v1", - scope_ref=query_reference, - ) - ], - now_ms=now_ms, - ) - - -def _materialize_promoted_finding_drifts( - conn: sqlite3.Connection, - evaluator: CanonicalPlanEvaluator, - *, - now_ms: int, -) -> None: - """Emit a new candidate when an accepted expected-count finding diverges.""" - for finding in list_assertion_claims( - conn, - kinds=(AssertionKind.FINDING,), - statuses=(AssertionStatus.ACCEPTED,), - ): - value = finding.value if isinstance(finding.value, dict) else {} - expected = value.get("expected") - query_reference = value.get("query_ref") - if not isinstance(expected, dict) or not isinstance(query_reference, str): - continue - query_hash = query_reference.removeprefix("query:") - query = get_query(conn, query_hash) - if query is None: - continue - evaluation = evaluator.evaluate( - QueryEvaluationRequest( - query=query, - purpose="finding-drift", - excluded_scope_refs=(f"assertion:{finding.assertion_id}",), - excluded_origin_prefixes=("notice.",), - ) - ) - if evaluation.cache_only or _matches_expected_count(expected, len(evaluation.member_refs)): - continue - current_id = f"finding-{hash_payload((query_hash, membership_merkle_root(evaluation.member_refs)))}" - current = get_result_set(conn, current_id) - if current is None: - current = put_result_set( - conn, - result_set_id=current_id, - query_hash=query_hash, - grain=evaluation.grain, - corpus_epoch=evaluation.corpus_epoch, - member_refs=evaluation.member_refs, - exactness=evaluation.exactness, - persistence_class="finding", - created_at_ms=now_ms, - ) - put_evaluation_receipt( - conn, - query_hash=query_hash, - receipt=evaluation.receipt, - result_set_id=current.result_set_id, - created_at_ms=now_ms, - ) - current_reference = result_set_ref(current.result_set_id).format() - upsert_findings_as_assertions( - conn, - [ - FindingAssertion( - claim_key="promoted-finding-expected-count-drift", - target_ref=f"assertion:{finding.assertion_id}", - body_text="Promoted finding no longer matches its expected member count.", - finding_kind="query-drift", - statistic={"op": "count", "value": current.member_count, "unit": "members"}, - n=current.member_count, - query_ref=query_reference, - result_set_ref=current_reference, - current_ref=current_reference, - expected=expected, - detector_ref="agent:standing-query-detector.v1", - scope_ref=f"assertion:{finding.assertion_id}", - ) - ], - now_ms=now_ms, - ) - - -def _has_promoted_expected_findings(conn: sqlite3.Connection) -> bool: - """Return whether convergence must run expected findings without a watch.""" - for finding in list_assertion_claims( - conn, - kinds=(AssertionKind.FINDING,), - statuses=(AssertionStatus.ACCEPTED,), - ): - value = finding.value if isinstance(finding.value, dict) else {} - if isinstance(value.get("expected"), dict): - return True - return False - - -def _matches_expected_count(expected: Mapping[str, object], actual: int) -> bool: - if expected.get("measure") != "member_count": - return True - value = expected.get("value") - if isinstance(value, bool) or not isinstance(value, int): - return True - op = expected.get("op") - if not isinstance(op, str): - return True - comparisons = { - "=": actual == value, - "!=": actual != value, - ">": actual > value, - ">=": actual >= value, - "<": actual < value, - "<=": actual <= value, - } - return comparisons.get(op, True) - - def make_default_convergence_stages(db_path: Path) -> tuple[ConvergenceStage, ...]: """Build the daemon's default post-ingest convergence stage set.""" return ( diff --git a/polylogue/daemon/convergence_standing_queries.py b/polylogue/daemon/convergence_standing_queries.py new file mode 100644 index 0000000000..66d2f418d4 --- /dev/null +++ b/polylogue/daemon/convergence_standing_queries.py @@ -0,0 +1,302 @@ +"""Session-scoped standing-query convergence stage. + +This module keeps the fourth convergence stage out of the historical +``convergence_stages`` hot file. The canonical-plan evaluator remains an +injected planner contract: durable identity JSON is provenance, not source +syntax to reverse-compile. +""" + +from __future__ import annotations + +import sqlite3 +import time +from collections.abc import Mapping, Sequence +from pathlib import Path + +from polylogue.archive.query.evaluator import CanonicalPlanEvaluator, QueryEvaluation, QueryEvaluationRequest +from polylogue.core.enums import AssertionKind, AssertionStatus +from polylogue.core.hashing import hash_payload +from polylogue.core.query_identity import query_ref, result_set_ref +from polylogue.daemon.convergence import ConvergenceStage, StageExecuteReturn +from polylogue.logging import get_logger +from polylogue.storage.sqlite.archive_tiers.user_write import ( + FindingAssertion, + list_assertion_claims, + upsert_findings_as_assertions, +) +from polylogue.storage.sqlite.connection_profile import open_daemon_connection +from polylogue.storage.sqlite.query_objects import ( + get_latest_result_set, + get_query, + get_result_set, + list_watched_queries, + membership_merkle_root, + put_evaluation_receipt, + put_result_set, +) + +logger = get_logger(__name__) + + +def make_standing_query_stage( + db_path: Path, + *, + evaluator: CanonicalPlanEvaluator | None = None, +) -> ConvergenceStage: + """Re-evaluate watched definitions after affected session convergence. + + The canonical-plan evaluator is deliberately injected. Identity JSON is + not an executable plan, so the stage stays inert until the owning planner + is supplied by broad runtime wiring. Its durable baseline, receipt, and + candidate-finding semantics are nevertheless the production path. + """ + + def check(_path: Path) -> bool: + return False + + def execute(_path: Path) -> StageExecuteReturn: + return True + + def check_sessions(session_ids: Sequence[str]) -> set[str]: + if evaluator is None or not session_ids: + return set() + user_db = _standing_user_db_path(db_path) + if not user_db.exists(): + return set() + try: + with sqlite3.connect(f"file:{user_db}?mode=ro", uri=True, timeout=5.0) as conn: + if list_watched_queries(conn) or _has_promoted_expected_findings(conn): + return set(session_ids) + return set() + except Exception: + logger.warning("standing-queries: watch lookup failed", exc_info=True) + return set(session_ids) + + def execute_sessions(session_ids: Sequence[str]) -> StageExecuteReturn: + if evaluator is None: + return True + user_db = _standing_user_db_path(db_path) + if not user_db.exists(): + return True + now_ms = int(time.time() * 1000) + ids = tuple(dict.fromkeys(str(session_id) for session_id in session_ids if session_id)) + try: + conn = open_daemon_connection(user_db, timeout=30.0) + try: + for query in list_watched_queries(conn): + query_reference = query_ref(query.query_hash).format() + evaluation = evaluator.evaluate( + QueryEvaluationRequest( + query=query, + purpose="standing-watch", + changed_session_ids=ids, + excluded_scope_refs=(query_reference,), + excluded_origin_prefixes=("notice.",), + ) + ) + if evaluation.cache_only: + # An index-only relation after reset is not evidence of + # membership drift. Compare durable user-tier baselines only. + continue + _materialize_watch_evaluation(conn, query.query_hash, evaluation, now_ms=now_ms) + _materialize_promoted_finding_drifts(conn, evaluator, now_ms=now_ms) + conn.commit() + finally: + conn.close() + return True + except Exception: + logger.warning("standing-queries: evaluation deferred", exc_info=True) + return False + + return ConvergenceStage( + name="standing-queries", + description="Re-evaluate watched query definitions and emit candidate deltas", + check=check, + execute=execute, + check_sessions=check_sessions, + execute_sessions=execute_sessions, + cpu_bound=False, + false_means_pending=True, + ) + + +def _standing_user_db_path(db_path: Path) -> Path: + return db_path.with_name("user.db") + + +def _watch_result_set_id(query_hash: str, member_refs: tuple[str, ...]) -> str: + return f"watch-{hash_payload((query_hash, membership_merkle_root(member_refs)))}" + + +def _materialize_watch_evaluation( + conn: sqlite3.Connection, + query_hash: str, + evaluation: QueryEvaluation, + *, + now_ms: int, +) -> None: + baseline = get_latest_result_set(conn, query_hash=query_hash, persistence_class="watch") + root = membership_merkle_root(evaluation.member_refs) + if baseline is not None and baseline.membership_merkle_root == root: + put_evaluation_receipt( + conn, + query_hash=query_hash, + receipt=evaluation.receipt, + result_set_id=baseline.result_set_id, + created_at_ms=now_ms, + ) + return + result_set_id = _watch_result_set_id(query_hash, evaluation.member_refs) + current = get_result_set(conn, result_set_id) + if current is None: + current = put_result_set( + conn, + result_set_id=result_set_id, + query_hash=query_hash, + grain=evaluation.grain, + corpus_epoch=evaluation.corpus_epoch, + member_refs=evaluation.member_refs, + exactness=evaluation.exactness, + persistence_class="watch", + created_at_ms=now_ms, + ) + put_evaluation_receipt( + conn, + query_hash=query_hash, + receipt=evaluation.receipt, + result_set_id=current.result_set_id, + created_at_ms=now_ms, + ) + if baseline is None: + return + query_reference = query_ref(query_hash).format() + current_reference = result_set_ref(current.result_set_id).format() + baseline_reference = result_set_ref(baseline.result_set_id).format() + upsert_findings_as_assertions( + conn, + [ + FindingAssertion( + claim_key="standing-query-membership-delta", + target_ref=query_reference, + body_text="Watched query membership changed after archive convergence.", + finding_kind="query-delta", + statistic={"op": "count", "value": current.member_count, "unit": "members"}, + n=current.member_count, + query_ref=query_reference, + result_set_ref=current_reference, + baseline_ref=baseline_reference, + current_ref=current_reference, + detector_ref="agent:standing-query-detector.v1", + scope_ref=query_reference, + ) + ], + now_ms=now_ms, + ) + + +def _materialize_promoted_finding_drifts( + conn: sqlite3.Connection, + evaluator: CanonicalPlanEvaluator, + *, + now_ms: int, +) -> None: + """Emit a new candidate when an accepted expected-count finding diverges.""" + for finding in list_assertion_claims( + conn, + kinds=(AssertionKind.FINDING,), + statuses=(AssertionStatus.ACCEPTED,), + ): + value = finding.value if isinstance(finding.value, dict) else {} + expected = value.get("expected") + query_reference = value.get("query_ref") + if not isinstance(expected, dict) or not isinstance(query_reference, str): + continue + query_hash = query_reference.removeprefix("query:") + query = get_query(conn, query_hash) + if query is None: + continue + evaluation = evaluator.evaluate( + QueryEvaluationRequest( + query=query, + purpose="finding-drift", + excluded_scope_refs=(f"assertion:{finding.assertion_id}",), + excluded_origin_prefixes=("notice.",), + ) + ) + if evaluation.cache_only or _matches_expected_count(expected, len(evaluation.member_refs)): + continue + current_id = f"finding-{hash_payload((query_hash, membership_merkle_root(evaluation.member_refs)))}" + current = get_result_set(conn, current_id) + if current is None: + current = put_result_set( + conn, + result_set_id=current_id, + query_hash=query_hash, + grain=evaluation.grain, + corpus_epoch=evaluation.corpus_epoch, + member_refs=evaluation.member_refs, + exactness=evaluation.exactness, + persistence_class="finding", + created_at_ms=now_ms, + ) + put_evaluation_receipt( + conn, + query_hash=query_hash, + receipt=evaluation.receipt, + result_set_id=current.result_set_id, + created_at_ms=now_ms, + ) + current_reference = result_set_ref(current.result_set_id).format() + upsert_findings_as_assertions( + conn, + [ + FindingAssertion( + claim_key="promoted-finding-expected-count-drift", + target_ref=f"assertion:{finding.assertion_id}", + body_text="Promoted finding no longer matches its expected member count.", + finding_kind="query-drift", + statistic={"op": "count", "value": current.member_count, "unit": "members"}, + n=current.member_count, + query_ref=query_reference, + result_set_ref=current_reference, + current_ref=current_reference, + expected=expected, + detector_ref="agent:standing-query-detector.v1", + scope_ref=f"assertion:{finding.assertion_id}", + ) + ], + now_ms=now_ms, + ) + + +def _has_promoted_expected_findings(conn: sqlite3.Connection) -> bool: + """Return whether convergence must run expected findings without a watch.""" + for finding in list_assertion_claims( + conn, + kinds=(AssertionKind.FINDING,), + statuses=(AssertionStatus.ACCEPTED,), + ): + value = finding.value if isinstance(finding.value, dict) else {} + if isinstance(value.get("expected"), dict): + return True + return False + + +def _matches_expected_count(expected: Mapping[str, object], actual: int) -> bool: + if expected.get("measure") != "member_count": + return True + value = expected.get("value") + if isinstance(value, bool) or not isinstance(value, int): + return True + op = expected.get("op") + if not isinstance(op, str): + return True + comparisons = { + "=": actual == value, + "!=": actual != value, + ">": actual > value, + ">=": actual >= value, + "<": actual < value, + "<=": actual <= value, + } + return comparisons.get(op, True) diff --git a/polylogue/storage/sqlite/query_objects.py b/polylogue/storage/sqlite/query_objects.py index c53d184826..3708d0da7b 100644 --- a/polylogue/storage/sqlite/query_objects.py +++ b/polylogue/storage/sqlite/query_objects.py @@ -216,15 +216,15 @@ def get_query(conn: sqlite3.Connection, query_hash: str) -> QueryObject | None: def list_watched_queries(conn: sqlite3.Connection) -> tuple[QueryObject, ...]: - """Return current mutable-name watch definitions, one name per query version.""" + """Return distinct immutable query versions selected by watched names.""" rows = conn.execute( """ - SELECT q.query_hash, q.canonical_plan_json, q.grain, q.lane, q.rank_policy, - q.definition_protocol_version + SELECT DISTINCT q.query_hash, q.canonical_plan_json, q.grain, q.lane, + q.rank_policy, q.definition_protocol_version FROM query_names AS n JOIN queries AS q ON q.query_hash = n.query_hash WHERE n.watch = 1 - ORDER BY n.updated_at_ms, n.name + ORDER BY q.query_hash """ ).fetchall() return tuple( diff --git a/tests/unit/daemon/test_standing_queries.py b/tests/unit/daemon/test_standing_queries.py index 588e6d43a8..f8dfebdca5 100644 --- a/tests/unit/daemon/test_standing_queries.py +++ b/tests/unit/daemon/test_standing_queries.py @@ -102,6 +102,23 @@ def test_cache_only_evaluation_after_index_reset_never_compares_baseline(tmp_pat assert conn.execute("SELECT COUNT(*) FROM result_sets WHERE persistence_class = 'watch'").fetchone()[0] == 1 +def test_watched_aliases_materialize_one_query_once_per_convergence(tmp_path: Path) -> None: + index_db, query_hash = _seed_watch(tmp_path) + with sqlite3.connect(tmp_path / "user.db") as conn: + put_query_name(conn, name="codex-alias", query_hash=query_hash, watch=True, updated_at_ms=3) + conn.commit() + + evaluator = _Evaluator(members=("session:one",)) + stage = make_standing_query_stage(index_db, evaluator=evaluator) + assert stage.execute_sessions is not None + assert stage.execute_sessions(("session:changed",)) is True + + assert len(evaluator.requests) == 1 + with sqlite3.connect(tmp_path / "user.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM result_sets WHERE persistence_class = 'watch'").fetchone()[0] == 1 + assert conn.execute("SELECT COUNT(*) FROM assertions WHERE kind = 'finding'").fetchone()[0] == 0 + + def test_promoted_expected_count_divergence_targets_original_finding_without_watch(tmp_path: Path) -> None: index_db, query_hash = _seed_watch(tmp_path, watch=False) with sqlite3.connect(tmp_path / "user.db") as conn: From 895b0693b786346f5b4deb41446ab994e4b137d6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 07:54:21 +0200 Subject: [PATCH 06/10] fix(query): preserve durable watch baselines Problem: immutable membership-addressed manifests could not encode the\nlast observed watch state when membership recurred, and receipt identities\ncould silently bind conflicting execution context.\n\nWhat changed: persist a per-query durable baseline pointer, validate receipt\nquery/result-set consistency and receipt-id idempotence, and fail closed at\nthe evaluation boundary for unsupported definition protocol versions.\n\nRef polylogue-rxdo.2\nRef polylogue-rxdo.5\nRef polylogue-rxdo.6\n\nCo-Authored-By: Claude --- polylogue/archive/query/evaluator.py | 4 + polylogue/core/query_identity.py | 11 +++ .../daemon/convergence_standing_queries.py | 17 +++- .../storage/sqlite/archive_tiers/user.py | 8 ++ .../user/008_query_evaluation_contracts.sql | 8 ++ polylogue/storage/sqlite/query_objects.py | 89 ++++++++++++++++--- tests/unit/archive/query/test_evaluator.py | 22 ++++- tests/unit/core/test_query_identity.py | 6 ++ tests/unit/daemon/test_standing_queries.py | 20 +++++ tests/unit/storage/test_query_objects.py | 76 ++++++++++++++++ 10 files changed, 244 insertions(+), 17 deletions(-) diff --git a/polylogue/archive/query/evaluator.py b/polylogue/archive/query/evaluator.py index 482ede940e..88dce272fe 100644 --- a/polylogue/archive/query/evaluator.py +++ b/polylogue/archive/query/evaluator.py @@ -13,6 +13,7 @@ from typing import Literal, Protocol from polylogue.archive.query.expression import RefOperand, RelationGrain, ResolvedRefOperand +from polylogue.core.query_identity import require_supported_definition_protocol_version from polylogue.core.refs import ObjectRef from polylogue.storage.sqlite.query_objects import ( EvaluationReceipt, @@ -39,6 +40,9 @@ class QueryEvaluationRequest: excluded_scope_refs: tuple[str, ...] = () excluded_origin_prefixes: tuple[str, ...] = () + def __post_init__(self) -> None: + require_supported_definition_protocol_version(self.query.definition_protocol_version) + @dataclass(frozen=True, slots=True) class QueryEvaluation: diff --git a/polylogue/core/query_identity.py b/polylogue/core/query_identity.py index 103c49d205..eab2e1c8aa 100644 --- a/polylogue/core/query_identity.py +++ b/polylogue/core/query_identity.py @@ -30,6 +30,9 @@ QUERY_RUN_ID_PREFIX: Final = "qr_" LEGACY_QUERY_DEFINITION_PROTOCOL_VERSION: Final = "polylogue.query-definition.v0" QUERY_DEFINITION_PROTOCOL_VERSION: Final = "polylogue.query-definition.v1" +SUPPORTED_QUERY_DEFINITION_PROTOCOL_VERSIONS: Final = frozenset( + {LEGACY_QUERY_DEFINITION_PROTOCOL_VERSION, QUERY_DEFINITION_PROTOCOL_VERSION} +) _COMMUTATIVE_OPERATORS: Final = frozenset({"and", "or"}) _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$") @@ -88,6 +91,14 @@ def query_hash_for_plan( ) +def require_supported_definition_protocol_version(definition_protocol_version: str) -> str: + """Return a supported definition version or fail closed before evaluation.""" + version = _nfc(definition_protocol_version) + if version not in SUPPORTED_QUERY_DEFINITION_PROTOCOL_VERSIONS: + raise ValueError(f"unsupported query definition protocol version: {version!r}") + return version + + def query_ref(query_hash: str) -> ObjectRef: """Build the registered public ref for a canonical query hash.""" _require_sha256(query_hash, label="query hash") diff --git a/polylogue/daemon/convergence_standing_queries.py b/polylogue/daemon/convergence_standing_queries.py index 66d2f418d4..a5f884996b 100644 --- a/polylogue/daemon/convergence_standing_queries.py +++ b/polylogue/daemon/convergence_standing_queries.py @@ -26,13 +26,14 @@ ) from polylogue.storage.sqlite.connection_profile import open_daemon_connection from polylogue.storage.sqlite.query_objects import ( - get_latest_result_set, get_query, get_result_set, + get_watched_query_baseline, list_watched_queries, membership_merkle_root, put_evaluation_receipt, put_result_set, + put_watched_query_baseline, ) logger = get_logger(__name__) @@ -135,7 +136,7 @@ def _materialize_watch_evaluation( *, now_ms: int, ) -> None: - baseline = get_latest_result_set(conn, query_hash=query_hash, persistence_class="watch") + baseline = get_watched_query_baseline(conn, query_hash) root = membership_merkle_root(evaluation.member_refs) if baseline is not None and baseline.membership_merkle_root == root: put_evaluation_receipt( @@ -145,6 +146,12 @@ def _materialize_watch_evaluation( result_set_id=baseline.result_set_id, created_at_ms=now_ms, ) + put_watched_query_baseline( + conn, + query_hash=query_hash, + result_set_id=baseline.result_set_id, + updated_at_ms=now_ms, + ) return result_set_id = _watch_result_set_id(query_hash, evaluation.member_refs) current = get_result_set(conn, result_set_id) @@ -167,6 +174,12 @@ def _materialize_watch_evaluation( result_set_id=current.result_set_id, created_at_ms=now_ms, ) + put_watched_query_baseline( + conn, + query_hash=query_hash, + result_set_id=current.result_set_id, + updated_at_ms=now_ms, + ) if baseline is None: return query_reference = query_ref(query_hash).format() diff --git a/polylogue/storage/sqlite/archive_tiers/user.py b/polylogue/storage/sqlite/archive_tiers/user.py index c143427437..bcbaccd3f1 100644 --- a/polylogue/storage/sqlite/archive_tiers/user.py +++ b/polylogue/storage/sqlite/archive_tiers/user.py @@ -129,6 +129,14 @@ CREATE INDEX IF NOT EXISTS idx_query_evaluation_receipts_query_time ON query_evaluation_receipts(query_hash, created_at_ms DESC); +-- Result manifests are immutable and membership-addressed, so a separate +-- pointer records the last watch evaluation even when membership recurs. +CREATE TABLE IF NOT EXISTS watched_query_baselines ( + query_hash TEXT PRIMARY KEY NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, + result_set_id TEXT NOT NULL REFERENCES result_sets(result_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT, + updated_at_ms INTEGER NOT NULL CHECK(updated_at_ms >= 0) +) STRICT; + -- Immutable versioned annotation construct definitions. Definition JSON is -- canonical and fingerprinted in Python; the row-level identity cannot be -- reused for a different construct after registration. diff --git a/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql b/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql index dcf07ed0cb..1e4119fc8f 100644 --- a/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql +++ b/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql @@ -31,3 +31,11 @@ CREATE TABLE IF NOT EXISTS query_evaluation_receipts ( ) STRICT; CREATE INDEX IF NOT EXISTS idx_query_evaluation_receipts_query_time ON query_evaluation_receipts(query_hash, created_at_ms DESC); + +-- Immutable manifests cannot express which recurring membership was most +-- recently evaluated, so watches retain an explicit durable baseline pointer. +CREATE TABLE IF NOT EXISTS watched_query_baselines ( + query_hash TEXT PRIMARY KEY NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, + result_set_id TEXT NOT NULL REFERENCES result_sets(result_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT, + updated_at_ms INTEGER NOT NULL CHECK(updated_at_ms >= 0) +) STRICT; diff --git a/polylogue/storage/sqlite/query_objects.py b/polylogue/storage/sqlite/query_objects.py index 3708d0da7b..5db3fa9e3b 100644 --- a/polylogue/storage/sqlite/query_objects.py +++ b/polylogue/storage/sqlite/query_objects.py @@ -14,6 +14,7 @@ canonical_query_plan, query_hash_for_plan, query_ref, + require_supported_definition_protocol_version, ) QueryEdgeKind = Literal["operand-of", "refines", "supersedes", "derived-from", "same-as"] @@ -82,6 +83,7 @@ def put_query( created_at_ms: int, ) -> QueryObject: """Idempotently persist one canonical expanded query plan.""" + definition_protocol_version = require_supported_definition_protocol_version(definition_protocol_version) canonical_plan = canonical_query_plan( planned_ast, grain=grain, @@ -280,6 +282,46 @@ def get_latest_result_set( return _manifest_from_row(row) if row is not None else None +def get_watched_query_baseline(conn: sqlite3.Connection, query_hash: str) -> ResultSetManifest | None: + """Return the last evaluated durable watch relation for one query.""" + row = conn.execute( + """ + SELECT rs.result_set_id, rs.query_hash, rs.grain, rs.corpus_epoch, rs.member_count, + rs.membership_merkle_root, rs.ordered_rank_hash, rs.exactness, rs.persistence_class + FROM watched_query_baselines AS baseline + JOIN result_sets AS rs ON rs.result_set_id = baseline.result_set_id + WHERE baseline.query_hash = ? + """, + (query_hash,), + ).fetchone() + return _manifest_from_row(row) if row is not None else None + + +def put_watched_query_baseline( + conn: sqlite3.Connection, + *, + query_hash: str, + result_set_id: str, + updated_at_ms: int, +) -> None: + """Advance one query's baseline pointer, including A→B→A transitions.""" + result_set = get_result_set(conn, result_set_id) + if result_set is None: + raise KeyError(f"result-set:{result_set_id}") + if result_set.query_hash != query_hash or result_set.persistence_class != "watch": + raise ValueError("watched query baseline must reference that query's watch result set") + conn.execute( + """ + INSERT INTO watched_query_baselines (query_hash, result_set_id, updated_at_ms) + VALUES (?, ?, ?) + ON CONFLICT(query_hash) DO UPDATE SET + result_set_id = excluded.result_set_id, + updated_at_ms = excluded.updated_at_ms + """, + (query_hash, result_set_id, updated_at_ms), + ) + + def put_retained_query_run( conn: sqlite3.Connection, *, @@ -323,6 +365,36 @@ def put_evaluation_receipt( created_at_ms: int, ) -> None: """Persist immutable execution context for a materialized relation.""" + if result_set_id is not None: + result_set = get_result_set(conn, result_set_id) + if result_set is None: + raise KeyError(f"result-set:{result_set_id}") + if result_set.query_hash != query_hash: + raise ValueError("evaluation receipt result set must belong to the same query") + values = ( + query_hash, + result_set_id, + receipt.source_generation, + receipt.user_generation, + receipt.index_generation, + receipt.runtime_build_ref, + _json(list(receipt.model_refs)), + _json(receipt.resolved_bounds or {}), + _json(receipt.degradation or {}), + created_at_ms, + ) + existing = conn.execute( + """ + SELECT query_hash, result_set_id, source_generation, user_generation, index_generation, + runtime_build_ref, model_refs_json, resolved_bounds_json, degradation_json, created_at_ms + FROM query_evaluation_receipts WHERE receipt_id = ? + """, + (receipt.receipt_id,), + ).fetchone() + if existing is not None: + if tuple(existing) != values: + raise ValueError(f"evaluation receipt id conflicts with a different execution: {receipt.receipt_id}") + return conn.execute( """ INSERT INTO query_evaluation_receipts ( @@ -330,21 +402,8 @@ def put_evaluation_receipt( index_generation, runtime_build_ref, model_refs_json, resolved_bounds_json, degradation_json, created_at_ms ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(receipt_id) DO NOTHING """, - ( - receipt.receipt_id, - query_hash, - result_set_id, - receipt.source_generation, - receipt.user_generation, - receipt.index_generation, - receipt.runtime_build_ref, - _json(list(receipt.model_refs)), - _json(receipt.resolved_bounds or {}), - _json(receipt.degradation or {}), - created_at_ms, - ), + (receipt.receipt_id, *values), ) @@ -463,6 +522,7 @@ def _manifest_from_row(row: sqlite3.Row | tuple[object, ...]) -> ResultSetManife "get_result_set", "get_result_set_members", "get_retained_query_run", + "get_watched_query_baseline", "list_watched_queries", "membership_merkle_root", "migrate_saved_query_assertions", @@ -472,4 +532,5 @@ def _manifest_from_row(row: sqlite3.Row | tuple[object, ...]) -> ResultSetManife "put_query_name", "put_retained_query_run", "put_result_set", + "put_watched_query_baseline", ] diff --git a/tests/unit/archive/query/test_evaluator.py b/tests/unit/archive/query/test_evaluator.py index 20beca6fe9..3a795576e9 100644 --- a/tests/unit/archive/query/test_evaluator.py +++ b/tests/unit/archive/query/test_evaluator.py @@ -14,7 +14,13 @@ from polylogue.core.refs import ObjectRef from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_tier from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier -from polylogue.storage.sqlite.query_objects import EvaluationReceipt, put_query, put_result_set, put_retained_query_run +from polylogue.storage.sqlite.query_objects import ( + EvaluationReceipt, + QueryObject, + put_query, + put_result_set, + put_retained_query_run, +) class _Evaluator: @@ -141,3 +147,17 @@ def test_retained_query_run_rejects_a_result_set_for_another_query() -> None: result_set_id=result.result_set_id, retained_at_ms=3, ) + + +def test_evaluation_request_rejects_an_unsupported_definition_protocol() -> None: + unsupported = QueryObject( + query_hash="a" * 64, + canonical_plan={}, + grain="session", + lane="dialogue", + rank_policy="mixed", + definition_protocol_version="polylogue.query-definition.v999", + ) + + with pytest.raises(ValueError, match="unsupported"): + QueryEvaluationRequest(query=unsupported, purpose="reference") diff --git a/tests/unit/core/test_query_identity.py b/tests/unit/core/test_query_identity.py index a5600b3bbd..1987f39bdf 100644 --- a/tests/unit/core/test_query_identity.py +++ b/tests/unit/core/test_query_identity.py @@ -8,6 +8,7 @@ query_hash_for_plan, query_ref, query_run_ref, + require_supported_definition_protocol_version, result_set_ref, ) @@ -87,3 +88,8 @@ def test_definition_protocol_version_is_bound_into_query_identity() -> None: rank_policy="mixed", definition_protocol_version="polylogue.query-definition.v2", ) + + +def test_unsupported_definition_protocol_version_fails_closed_at_execution_boundary() -> None: + with pytest.raises(ValueError, match="unsupported"): + require_supported_definition_protocol_version("polylogue.query-definition.v999") diff --git a/tests/unit/daemon/test_standing_queries.py b/tests/unit/daemon/test_standing_queries.py index f8dfebdca5..c08b1510d7 100644 --- a/tests/unit/daemon/test_standing_queries.py +++ b/tests/unit/daemon/test_standing_queries.py @@ -119,6 +119,26 @@ def test_watched_aliases_materialize_one_query_once_per_convergence(tmp_path: Pa assert conn.execute("SELECT COUNT(*) FROM assertions WHERE kind = 'finding'").fetchone()[0] == 0 +def test_watch_baseline_tracks_return_to_a_prior_membership(tmp_path: Path) -> None: + index_db, _ = _seed_watch(tmp_path) + evaluator = _Evaluator(members=("session:a",)) + stage = make_standing_query_stage(index_db, evaluator=evaluator) + assert stage.execute_sessions is not None + assert stage.execute_sessions(("session:changed",)) is True # baseline A + evaluator.members = ("session:b",) + assert stage.execute_sessions(("session:changed",)) is True # A -> B + evaluator.members = ("session:a",) + assert stage.execute_sessions(("session:changed",)) is True # B -> A + + with sqlite3.connect(tmp_path / "user.db") as conn: + rows = conn.execute( + "SELECT value_json FROM assertions WHERE kind = 'finding' ORDER BY created_at_ms, assertion_id" + ).fetchall() + assert len(rows) == 2 + assert "baseline_ref" in str(rows[1][0]) + assert "result-set:watch-" in str(rows[1][0]) + + def test_promoted_expected_count_divergence_targets_original_finding_without_watch(tmp_path: Path) -> None: index_db, query_hash = _seed_watch(tmp_path, watch=False) with sqlite3.connect(tmp_path / "user.db") as conn: diff --git a/tests/unit/storage/test_query_objects.py b/tests/unit/storage/test_query_objects.py index 7737ed89ef..22f19b8bea 100644 --- a/tests/unit/storage/test_query_objects.py +++ b/tests/unit/storage/test_query_objects.py @@ -11,6 +11,7 @@ EvaluationReceipt, get_query, get_retained_query_run, + get_watched_query_baseline, list_watched_queries, migrate_saved_query_assertions, put_evaluation_receipt, @@ -19,6 +20,7 @@ put_query_name, put_result_set, put_retained_query_run, + put_watched_query_baseline, ) @@ -173,3 +175,77 @@ def test_watched_definition_retention_and_receipt_are_durable_contracts() -> Non assert list_watched_queries(conn) == (restored,) assert get_retained_query_run(conn, "qr_saved") is not None assert conn.execute("SELECT result_set_id FROM query_evaluation_receipts").fetchone()[0] == result.result_set_id + + +def test_watch_baseline_and_receipt_ids_reject_cross_query_or_conflicting_state() -> None: + conn = _conn() + first = put_query( + conn, + {"field": "title", "value": "first"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + second = put_query( + conn, + {"field": "title", "value": "second"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + first_result = put_result_set( + conn, + result_set_id="first-watch", + query_hash=first.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:one",), + exactness="exact", + persistence_class="watch", + created_at_ms=2, + ) + second_result = put_result_set( + conn, + result_set_id="second-watch", + query_hash=second.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:two",), + exactness="exact", + persistence_class="watch", + created_at_ms=2, + ) + put_watched_query_baseline( + conn, query_hash=first.query_hash, result_set_id=first_result.result_set_id, updated_at_ms=2 + ) + assert get_watched_query_baseline(conn, first.query_hash) == first_result + with pytest.raises(ValueError, match="baseline"): + put_watched_query_baseline( + conn, query_hash=first.query_hash, result_set_id=second_result.result_set_id, updated_at_ms=2 + ) + + receipt = EvaluationReceipt("receipt-stable", "source:g1", "user:g1", "index:g1", "build:test") + put_evaluation_receipt( + conn, query_hash=first.query_hash, result_set_id=first_result.result_set_id, receipt=receipt, created_at_ms=3 + ) + put_evaluation_receipt( + conn, query_hash=first.query_hash, result_set_id=first_result.result_set_id, receipt=receipt, created_at_ms=3 + ) + with pytest.raises(ValueError, match="same query"): + put_evaluation_receipt( + conn, + query_hash=first.query_hash, + result_set_id=second_result.result_set_id, + receipt=receipt, + created_at_ms=3, + ) + with pytest.raises(ValueError, match="conflicts"): + put_evaluation_receipt( + conn, + query_hash=first.query_hash, + result_set_id=first_result.result_set_id, + receipt=EvaluationReceipt("receipt-stable", "source:g2", "user:g1", "index:g1", "build:test"), + created_at_ms=3, + ) From 1b6ee8b9658ecaf91f905991f76f8b4e92769a24 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 08:10:05 +0200 Subject: [PATCH 07/10] fix(query): enforce retained run immutability Problem: a retained run could be rebound to another execution, fresh user\nDDL drifted from the additive v8 migration, and watch baseline regressions\ndid not prove the final pointer state.\n\nWhat changed: make retained run ids immutable after creation, compare all\nquery-provenance schema objects semantically across migrated and fresh tiers,\nand prove durable baseline transitions through a repeated membership cycle.\n\nRef polylogue-rxdo.2\nRef polylogue-rxdo.5\nRef polylogue-rxdo.6\n\nCo-Authored-By: Claude --- .../storage/sqlite/archive_tiers/user.py | 9 +-- polylogue/storage/sqlite/query_objects.py | 12 ++-- tests/unit/archive/query/test_evaluator.py | 58 +++++++++++++++++++ tests/unit/daemon/test_standing_queries.py | 23 +++++++- tests/unit/storage/test_durable_migrations.py | 42 ++++++++++---- 5 files changed, 123 insertions(+), 21 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/user.py b/polylogue/storage/sqlite/archive_tiers/user.py index bcbaccd3f1..98b3063e95 100644 --- a/polylogue/storage/sqlite/archive_tiers/user.py +++ b/polylogue/storage/sqlite/archive_tiers/user.py @@ -50,16 +50,17 @@ grain TEXT NOT NULL CHECK(length(trim(grain)) > 0), lane TEXT NOT NULL CHECK(length(trim(lane)) > 0), rank_policy TEXT NOT NULL CHECK(length(trim(rank_policy)) > 0), - definition_protocol_version TEXT NOT NULL CHECK(length(trim(definition_protocol_version)) > 0), - created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0) + created_at_ms INTEGER NOT NULL CHECK(created_at_ms >= 0), + definition_protocol_version TEXT NOT NULL DEFAULT 'polylogue.query-definition.v0' + CHECK(length(trim(definition_protocol_version)) > 0) ) STRICT; CREATE TABLE IF NOT EXISTS query_names ( name TEXT PRIMARY KEY NOT NULL CHECK(length(trim(name)) > 0), query_hash TEXT NOT NULL REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, supersedes_query_hash TEXT REFERENCES queries(query_hash) ON UPDATE RESTRICT ON DELETE RESTRICT, - watch INTEGER NOT NULL DEFAULT 0 CHECK(watch IN (0, 1)), - updated_at_ms INTEGER NOT NULL CHECK(updated_at_ms >= 0) + updated_at_ms INTEGER NOT NULL CHECK(updated_at_ms >= 0), + watch INTEGER NOT NULL DEFAULT 0 CHECK(watch IN (0, 1)) ) STRICT; CREATE INDEX IF NOT EXISTS idx_query_names_query_hash diff --git a/polylogue/storage/sqlite/query_objects.py b/polylogue/storage/sqlite/query_objects.py index 5db3fa9e3b..4c7f59e546 100644 --- a/polylogue/storage/sqlite/query_objects.py +++ b/polylogue/storage/sqlite/query_objects.py @@ -335,14 +335,18 @@ def put_retained_query_run( raise KeyError(f"result-set:{result_set_id}") if result_set.query_hash != query_hash: raise ValueError("retained query run result set must belong to the same query") + existing = conn.execute( + "SELECT query_hash, result_set_id, retained_at_ms FROM retained_query_runs WHERE run_id = ?", + (run_id,), + ).fetchone() + if existing is not None: + if tuple(existing) == (query_hash, result_set_id, retained_at_ms): + return RetainedQueryRun(run_id, query_hash, result_set_id) + raise ValueError(f"retained query run id conflicts with a different execution: {run_id}") conn.execute( """ INSERT INTO retained_query_runs (run_id, query_hash, result_set_id, retained_at_ms) VALUES (?, ?, ?, ?) - ON CONFLICT(run_id) DO UPDATE SET - query_hash = excluded.query_hash, - result_set_id = excluded.result_set_id, - retained_at_ms = excluded.retained_at_ms """, (run_id, query_hash, result_set_id, retained_at_ms), ) diff --git a/tests/unit/archive/query/test_evaluator.py b/tests/unit/archive/query/test_evaluator.py index 3a795576e9..602e173872 100644 --- a/tests/unit/archive/query/test_evaluator.py +++ b/tests/unit/archive/query/test_evaluator.py @@ -149,6 +149,64 @@ def test_retained_query_run_rejects_a_result_set_for_another_query() -> None: ) +def test_retained_query_run_id_cannot_rebind_to_a_different_execution() -> None: + conn = _conn() + first = put_query( + conn, + {"field": "title", "value": "first"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + second = put_query( + conn, + {"field": "title", "value": "second"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + first_result = put_result_set( + conn, + result_set_id="first-retained", + query_hash=first.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:first",), + exactness="exact", + persistence_class="pinned", + created_at_ms=2, + ) + second_result = put_result_set( + conn, + result_set_id="second-retained", + query_hash=second.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:second",), + exactness="exact", + persistence_class="pinned", + created_at_ms=2, + ) + put_retained_query_run( + conn, + run_id="qr_immutable", + query_hash=first.query_hash, + result_set_id=first_result.result_set_id, + retained_at_ms=3, + ) + + with pytest.raises(ValueError, match="conflicts"): + put_retained_query_run( + conn, + run_id="qr_immutable", + query_hash=second.query_hash, + result_set_id=second_result.result_set_id, + retained_at_ms=4, + ) + + def test_evaluation_request_rejects_an_unsupported_definition_protocol() -> None: unsupported = QueryObject( query_hash="a" * 64, diff --git a/tests/unit/daemon/test_standing_queries.py b/tests/unit/daemon/test_standing_queries.py index c08b1510d7..4e1f4c5121 100644 --- a/tests/unit/daemon/test_standing_queries.py +++ b/tests/unit/daemon/test_standing_queries.py @@ -14,7 +14,13 @@ mark_assertion_status, upsert_findings_as_assertions, ) -from polylogue.storage.sqlite.query_objects import EvaluationReceipt, put_query, put_query_name +from polylogue.storage.sqlite.query_objects import ( + EvaluationReceipt, + get_result_set_members, + get_watched_query_baseline, + put_query, + put_query_name, +) class _Evaluator: @@ -120,7 +126,7 @@ def test_watched_aliases_materialize_one_query_once_per_convergence(tmp_path: Pa def test_watch_baseline_tracks_return_to_a_prior_membership(tmp_path: Path) -> None: - index_db, _ = _seed_watch(tmp_path) + index_db, query_hash = _seed_watch(tmp_path) evaluator = _Evaluator(members=("session:a",)) stage = make_standing_query_stage(index_db, evaluator=evaluator) assert stage.execute_sessions is not None @@ -137,6 +143,19 @@ def test_watch_baseline_tracks_return_to_a_prior_membership(tmp_path: Path) -> N assert len(rows) == 2 assert "baseline_ref" in str(rows[1][0]) assert "result-set:watch-" in str(rows[1][0]) + baseline = get_watched_query_baseline(conn, query_hash) + assert baseline is not None + assert get_result_set_members(conn, baseline.result_set_id) == ("session:a",) + + evaluator.members = ("session:b",) + assert stage.execute_sessions(("session:changed",)) is True # A -> B again + with sqlite3.connect(tmp_path / "user.db") as conn: + # Stable finding identity deduplicates the repeated B delta, but the + # durable pointer must still advance to B for the next transition. + assert conn.execute("SELECT COUNT(*) FROM assertions WHERE kind = 'finding'").fetchone()[0] == 2 + baseline = get_watched_query_baseline(conn, query_hash) + assert baseline is not None + assert get_result_set_members(conn, baseline.result_set_id) == ("session:b",) def test_promoted_expected_count_divergence_targets_original_finding_without_watch(tmp_path: Path) -> None: diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index 9a34539ac0..e9ac9d7cdf 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -3,6 +3,7 @@ import hashlib import json import os +import re import shutil import sqlite3 from collections.abc import Callable @@ -137,29 +138,48 @@ def _create_user_v5(path: Path) -> None: conn.close() -_USER_V6_SCHEMA_OBJECTS = ( +_USER_DURABLE_SCHEMA_OBJECTS = ( "annotation_batches", "annotation_schemas", "idx_annotation_batches_schema_target_time", "idx_annotation_batches_source_result_time", "idx_assertions_scope_kind_status", + "queries", + "query_names", + "idx_query_names_query_hash", + "idx_query_names_watch", + "result_sets", + "idx_result_sets_query_epoch", + "result_set_members", + "query_edges", + "idx_query_edges_dst_kind", + "retained_query_runs", + "query_evaluation_receipts", + "idx_query_evaluation_receipts_query_time", + "watched_query_baselines", ) -def _user_v6_schema_sql(conn: sqlite3.Connection) -> tuple[tuple[object, ...], ...]: - placeholders = ",".join("?" for _ in _USER_V6_SCHEMA_OBJECTS) - return tuple( - tuple(row) - for row in conn.execute( - f""" +def _user_durable_schema_sql(conn: sqlite3.Connection) -> tuple[tuple[object, ...], ...]: + placeholders = ",".join("?" for _ in _USER_DURABLE_SCHEMA_OBJECTS) + rows = conn.execute( + f""" SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE name IN ({placeholders}) ORDER BY type, name """, - _USER_V6_SCHEMA_OBJECTS, - ) - ) + _USER_DURABLE_SCHEMA_OBJECTS, + ).fetchall() + return tuple((str(row[0]), str(row[1]), str(row[2]), _normalize_schema_sql(str(row[3]))) for row in rows) + + +def _normalize_schema_sql(sql: str) -> str: + """Compare SQLite DDL semantics despite ALTER TABLE's punctuation layout.""" + collapsed = re.sub(r"\s+", " ", sql).strip() + collapsed = re.sub(r"\s*,\s*", ",", collapsed) + collapsed = re.sub(r"\(\s*", "(", collapsed) + return re.sub(r"\s*\)", ")", collapsed) def _assert_user_v6_annotation_checks(conn: sqlite3.Connection, *, suffix: str) -> None: @@ -350,7 +370,7 @@ def test_user_tier_v5_annotation_migration_requires_verified_backup_and_matches_ fresh_db = tmp_path / "fresh-user-v6.db" initialize_archive_database(fresh_db, ArchiveTier.USER) with sqlite3.connect(fresh_db) as fresh_conn: - assert _user_v6_schema_sql(conn) == _user_v6_schema_sql(fresh_conn) + assert _user_durable_schema_sql(conn) == _user_durable_schema_sql(fresh_conn) assert tuple(conn.execute("PRAGMA foreign_key_list(annotation_batches)")) == tuple( fresh_conn.execute("PRAGMA foreign_key_list(annotation_batches)") ) From f2c46058f267699893be013f5d5a00d1778cb3e1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 08:20:37 +0200 Subject: [PATCH 08/10] fix(query): bind durable snapshots to evaluations Problem: membership-addressed watch manifests reused stale epoch or ranking\nmetadata, while SQL-level writers could bypass same-query provenance checks.\n\nWhat changed: include every persisted snapshot field in watch-manifest\nidentity, update baselines without false membership deltas, enforce matching\nquery/result-set pairs with durable triggers, and make the migration runner\ntrigger-aware.\n\nRef polylogue-rxdo.2\nRef polylogue-rxdo.5\nRef polylogue-rxdo.6\n\nCo-Authored-By: Claude --- .../daemon/convergence_standing_queries.py | 30 +++++++++-- .../storage/sqlite/archive_tiers/user.py | 39 ++++++++++++++ polylogue/storage/sqlite/migration_runner.py | 12 +++-- .../user/008_query_evaluation_contracts.sql | 39 ++++++++++++++ tests/unit/daemon/test_standing_queries.py | 24 ++++++++- tests/unit/storage/test_durable_migrations.py | 51 +++++++++++++++++++ tests/unit/storage/test_query_objects.py | 21 ++++++++ 7 files changed, 207 insertions(+), 9 deletions(-) diff --git a/polylogue/daemon/convergence_standing_queries.py b/polylogue/daemon/convergence_standing_queries.py index a5f884996b..c5a8379c1f 100644 --- a/polylogue/daemon/convergence_standing_queries.py +++ b/polylogue/daemon/convergence_standing_queries.py @@ -125,8 +125,18 @@ def _standing_user_db_path(db_path: Path) -> Path: return db_path.with_name("user.db") -def _watch_result_set_id(query_hash: str, member_refs: tuple[str, ...]) -> str: - return f"watch-{hash_payload((query_hash, membership_merkle_root(member_refs)))}" +def _watch_result_set_id(query_hash: str, evaluation: QueryEvaluation) -> str: + """Name an immutable watch snapshot by every persisted manifest field.""" + snapshot = ( + query_hash, + evaluation.grain, + evaluation.corpus_epoch, + membership_merkle_root(evaluation.member_refs), + hash_payload(list(evaluation.member_refs)), + evaluation.exactness, + "watch", + ) + return f"watch-{hash_payload(snapshot)}" def _materialize_watch_evaluation( @@ -138,7 +148,17 @@ def _materialize_watch_evaluation( ) -> None: baseline = get_watched_query_baseline(conn, query_hash) root = membership_merkle_root(evaluation.member_refs) - if baseline is not None and baseline.membership_merkle_root == root: + rank_hash = hash_payload(list(evaluation.member_refs)) + same_snapshot = baseline is not None and ( + baseline.grain == evaluation.grain + and baseline.corpus_epoch == evaluation.corpus_epoch + and baseline.membership_merkle_root == root + and baseline.ordered_rank_hash == rank_hash + and baseline.exactness == evaluation.exactness + and baseline.persistence_class == "watch" + ) + if same_snapshot: + assert baseline is not None put_evaluation_receipt( conn, query_hash=query_hash, @@ -153,7 +173,7 @@ def _materialize_watch_evaluation( updated_at_ms=now_ms, ) return - result_set_id = _watch_result_set_id(query_hash, evaluation.member_refs) + result_set_id = _watch_result_set_id(query_hash, evaluation) current = get_result_set(conn, result_set_id) if current is None: current = put_result_set( @@ -180,7 +200,7 @@ def _materialize_watch_evaluation( result_set_id=current.result_set_id, updated_at_ms=now_ms, ) - if baseline is None: + if baseline is None or baseline.membership_merkle_root == root: return query_reference = query_ref(query_hash).format() current_reference = result_set_ref(current.result_set_id).format() diff --git a/polylogue/storage/sqlite/archive_tiers/user.py b/polylogue/storage/sqlite/archive_tiers/user.py index 98b3063e95..94111664a0 100644 --- a/polylogue/storage/sqlite/archive_tiers/user.py +++ b/polylogue/storage/sqlite/archive_tiers/user.py @@ -138,6 +138,45 @@ updated_at_ms INTEGER NOT NULL CHECK(updated_at_ms >= 0) ) STRICT; +CREATE TRIGGER retained_query_runs_result_set_query_match_insert +BEFORE INSERT ON retained_query_runs +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'retained query run result set must belong to the same query'); +END; +CREATE TRIGGER retained_query_runs_result_set_query_match_update +BEFORE UPDATE OF query_hash, result_set_id ON retained_query_runs +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'retained query run result set must belong to the same query'); +END; +CREATE TRIGGER query_evaluation_receipts_result_set_query_match_insert +BEFORE INSERT ON query_evaluation_receipts +WHEN NEW.result_set_id IS NOT NULL + AND (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'evaluation receipt result set must belong to the same query'); +END; +CREATE TRIGGER query_evaluation_receipts_result_set_query_match_update +BEFORE UPDATE OF query_hash, result_set_id ON query_evaluation_receipts +WHEN NEW.result_set_id IS NOT NULL + AND (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'evaluation receipt result set must belong to the same query'); +END; +CREATE TRIGGER watched_query_baselines_result_set_query_match_insert +BEFORE INSERT ON watched_query_baselines +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'watched query baseline result set must belong to the same query'); +END; +CREATE TRIGGER watched_query_baselines_result_set_query_match_update +BEFORE UPDATE OF query_hash, result_set_id ON watched_query_baselines +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'watched query baseline result set must belong to the same query'); +END; + -- Immutable versioned annotation construct definitions. Definition JSON is -- canonical and fingerprinted in Python; the row-level identity cannot be -- reused for a different construct after registration. diff --git a/polylogue/storage/sqlite/migration_runner.py b/polylogue/storage/sqlite/migration_runner.py index 6c7c0e7435..e12ba1e869 100644 --- a/polylogue/storage/sqlite/migration_runner.py +++ b/polylogue/storage/sqlite/migration_runner.py @@ -481,9 +481,15 @@ def validate_migration_backup_manifest( def _execute_migration_sql(conn: sqlite3.Connection, sql: str) -> None: - statements = [statement.strip() for statement in sql.split(";") if statement.strip()] - for statement in statements: - conn.execute(statement) + statement = "" + for line in sql.splitlines(keepends=True): + statement += line + if sqlite3.complete_statement(statement): + if statement.strip(): + conn.execute(statement) + statement = "" + if statement.strip(): + raise MigrationError("migration SQL ended with an incomplete statement") def migrate_archive_tier( diff --git a/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql b/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql index 1e4119fc8f..98ac8bef00 100644 --- a/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql +++ b/polylogue/storage/sqlite/migrations/user/008_query_evaluation_contracts.sql @@ -39,3 +39,42 @@ CREATE TABLE IF NOT EXISTS watched_query_baselines ( result_set_id TEXT NOT NULL REFERENCES result_sets(result_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT, updated_at_ms INTEGER NOT NULL CHECK(updated_at_ms >= 0) ) STRICT; + +CREATE TRIGGER retained_query_runs_result_set_query_match_insert +BEFORE INSERT ON retained_query_runs +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'retained query run result set must belong to the same query'); +END; +CREATE TRIGGER retained_query_runs_result_set_query_match_update +BEFORE UPDATE OF query_hash, result_set_id ON retained_query_runs +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'retained query run result set must belong to the same query'); +END; +CREATE TRIGGER query_evaluation_receipts_result_set_query_match_insert +BEFORE INSERT ON query_evaluation_receipts +WHEN NEW.result_set_id IS NOT NULL + AND (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'evaluation receipt result set must belong to the same query'); +END; +CREATE TRIGGER query_evaluation_receipts_result_set_query_match_update +BEFORE UPDATE OF query_hash, result_set_id ON query_evaluation_receipts +WHEN NEW.result_set_id IS NOT NULL + AND (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'evaluation receipt result set must belong to the same query'); +END; +CREATE TRIGGER watched_query_baselines_result_set_query_match_insert +BEFORE INSERT ON watched_query_baselines +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'watched query baseline result set must belong to the same query'); +END; +CREATE TRIGGER watched_query_baselines_result_set_query_match_update +BEFORE UPDATE OF query_hash, result_set_id ON watched_query_baselines +WHEN (SELECT query_hash FROM result_sets WHERE result_set_id = NEW.result_set_id) IS NOT NEW.query_hash +BEGIN + SELECT RAISE(ABORT, 'watched query baseline result set must belong to the same query'); +END; diff --git a/tests/unit/daemon/test_standing_queries.py b/tests/unit/daemon/test_standing_queries.py index 4e1f4c5121..fe5a12e47a 100644 --- a/tests/unit/daemon/test_standing_queries.py +++ b/tests/unit/daemon/test_standing_queries.py @@ -27,6 +27,7 @@ class _Evaluator: def __init__(self, *, members: tuple[str, ...], cache_only: bool = False) -> None: self.members = members self.cache_only = cache_only + self.corpus_epoch = "index:g1" self.requests: list[QueryEvaluationRequest] = [] def evaluate(self, request: QueryEvaluationRequest) -> QueryEvaluation: @@ -34,7 +35,7 @@ def evaluate(self, request: QueryEvaluationRequest) -> QueryEvaluation: return QueryEvaluation( grain="session", member_refs=self.members, - corpus_epoch="index:g1", + corpus_epoch=self.corpus_epoch, exactness="exact", cache_only=self.cache_only, receipt=EvaluationReceipt( @@ -158,6 +159,27 @@ def test_watch_baseline_tracks_return_to_a_prior_membership(tmp_path: Path) -> N assert get_result_set_members(conn, baseline.result_set_id) == ("session:b",) +def test_watch_snapshot_metadata_creates_a_new_manifest_without_membership_drift(tmp_path: Path) -> None: + index_db, query_hash = _seed_watch(tmp_path) + evaluator = _Evaluator(members=("session:a", "session:b")) + stage = make_standing_query_stage(index_db, evaluator=evaluator) + assert stage.execute_sessions is not None + assert stage.execute_sessions(("session:changed",)) is True + + evaluator.members = ("session:b", "session:a") # same membership, new rank order + assert stage.execute_sessions(("session:changed",)) is True + evaluator.corpus_epoch = "index:g2" + assert stage.execute_sessions(("session:changed",)) is True + + with sqlite3.connect(tmp_path / "user.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM result_sets WHERE persistence_class = 'watch'").fetchone()[0] == 3 + assert conn.execute("SELECT COUNT(*) FROM assertions WHERE kind = 'finding'").fetchone()[0] == 0 + baseline = get_watched_query_baseline(conn, query_hash) + assert baseline is not None + assert baseline.corpus_epoch == "index:g2" + assert get_result_set_members(conn, baseline.result_set_id) == ("session:b", "session:a") + + def test_promoted_expected_count_divergence_targets_original_finding_without_watch(tmp_path: Path) -> None: index_db, query_hash = _seed_watch(tmp_path, watch=False) with sqlite3.connect(tmp_path / "user.db") as conn: diff --git a/tests/unit/storage/test_durable_migrations.py b/tests/unit/storage/test_durable_migrations.py index e9ac9d7cdf..30fc78185a 100644 --- a/tests/unit/storage/test_durable_migrations.py +++ b/tests/unit/storage/test_durable_migrations.py @@ -157,6 +157,12 @@ def _create_user_v5(path: Path) -> None: "query_evaluation_receipts", "idx_query_evaluation_receipts_query_time", "watched_query_baselines", + "retained_query_runs_result_set_query_match_insert", + "retained_query_runs_result_set_query_match_update", + "query_evaluation_receipts_result_set_query_match_insert", + "query_evaluation_receipts_result_set_query_match_update", + "watched_query_baselines_result_set_query_match_insert", + "watched_query_baselines_result_set_query_match_update", ) @@ -182,6 +188,49 @@ def _normalize_schema_sql(sql: str) -> str: return re.sub(r"\s*\)", ")", collapsed) +def _assert_query_provenance_binding_triggers(conn: sqlite3.Connection) -> None: + """Exercise migration and fresh DDL against raw SQL bypasses.""" + first_hash, second_hash = "b" * 64, "c" * 64 + conn.executemany( + """ + INSERT INTO queries ( + query_hash, canonical_plan_json, grain, lane, rank_policy, created_at_ms + ) VALUES (?, '{}', 'session', 'dialogue', 'mixed', 1) + """, + ((first_hash,), (second_hash,)), + ) + conn.executemany( + """ + INSERT INTO result_sets ( + result_set_id, query_hash, grain, corpus_epoch, member_count, + membership_merkle_root, ordered_rank_hash, exactness, persistence_class, created_at_ms + ) VALUES (?, ?, 'session', 'index:g1', 0, ?, ?, 'exact', 'watch', 1) + """, + (("binding-first", first_hash, "1" * 64, "2" * 64), ("binding-second", second_hash, "3" * 64, "4" * 64)), + ) + with pytest.raises(sqlite3.IntegrityError, match="same query"): + conn.execute( + "INSERT INTO retained_query_runs (run_id, query_hash, result_set_id, retained_at_ms) VALUES ('qr_raw', ?, 'binding-second', 1)", + (first_hash,), + ) + with pytest.raises(sqlite3.IntegrityError, match="same query"): + conn.execute( + """ + INSERT INTO query_evaluation_receipts ( + receipt_id, query_hash, result_set_id, source_generation, user_generation, + index_generation, runtime_build_ref, model_refs_json, resolved_bounds_json, + degradation_json, created_at_ms + ) VALUES ('receipt-raw', ?, 'binding-second', 's', 'u', 'i', 'b', '[]', '{}', '{}', 1) + """, + (first_hash,), + ) + with pytest.raises(sqlite3.IntegrityError, match="same query"): + conn.execute( + "INSERT INTO watched_query_baselines (query_hash, result_set_id, updated_at_ms) VALUES (?, 'binding-second', 1)", + (first_hash,), + ) + + def _assert_user_v6_annotation_checks(conn: sqlite3.Connection, *, suffix: str) -> None: with pytest.raises(sqlite3.IntegrityError): conn.execute( @@ -366,11 +415,13 @@ def test_user_tier_v5_annotation_migration_requires_verified_backup_and_matches_ assert {str(row[1]) for row in conn.execute("PRAGMA index_list(assertions)")} >= { "idx_assertions_scope_kind_status" } + _assert_query_provenance_binding_triggers(conn) fresh_db = tmp_path / "fresh-user-v6.db" initialize_archive_database(fresh_db, ArchiveTier.USER) with sqlite3.connect(fresh_db) as fresh_conn: assert _user_durable_schema_sql(conn) == _user_durable_schema_sql(fresh_conn) + _assert_query_provenance_binding_triggers(fresh_conn) assert tuple(conn.execute("PRAGMA foreign_key_list(annotation_batches)")) == tuple( fresh_conn.execute("PRAGMA foreign_key_list(annotation_batches)") ) diff --git a/tests/unit/storage/test_query_objects.py b/tests/unit/storage/test_query_objects.py index 22f19b8bea..5be268d23c 100644 --- a/tests/unit/storage/test_query_objects.py +++ b/tests/unit/storage/test_query_objects.py @@ -249,3 +249,24 @@ def test_watch_baseline_and_receipt_ids_reject_cross_query_or_conflicting_state( receipt=EvaluationReceipt("receipt-stable", "source:g2", "user:g1", "index:g1", "build:test"), created_at_ms=3, ) + with pytest.raises(sqlite3.IntegrityError, match="same query"): + conn.execute( + "INSERT INTO retained_query_runs (run_id, query_hash, result_set_id, retained_at_ms) VALUES (?, ?, ?, ?)", + ("qr_raw_mismatch", first.query_hash, second_result.result_set_id, 4), + ) + with pytest.raises(sqlite3.IntegrityError, match="same query"): + conn.execute( + """ + INSERT INTO query_evaluation_receipts ( + receipt_id, query_hash, result_set_id, source_generation, user_generation, + index_generation, runtime_build_ref, model_refs_json, resolved_bounds_json, + degradation_json, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, '[]', '{}', '{}', ?) + """, + ("receipt-raw-mismatch", first.query_hash, second_result.result_set_id, "s", "u", "i", "b", 4), + ) + with pytest.raises(sqlite3.IntegrityError, match="same query"): + conn.execute( + "INSERT INTO watched_query_baselines (query_hash, result_set_id, updated_at_ms) VALUES (?, ?, ?)", + (first.query_hash, second_result.result_set_id, 4), + ) From 71b3ceb1afcaad8adacc6116ae9d9a03e80f75fe Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 08:22:29 +0200 Subject: [PATCH 09/10] fix(query): reject inexact retained set operands Problem: a sampled or capped durable relation could be used as an exact\nset-algebra operand despite incomplete membership.\n\nWhat changed: make the retained-reference resolver fail closed unless the\nmanifest is exact, with a regression covering sampled snapshots.\n\nRef polylogue-rxdo.6\n\nCo-Authored-By: Claude --- polylogue/archive/query/evaluator.py | 4 +++ tests/unit/archive/query/test_evaluator.py | 29 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/polylogue/archive/query/evaluator.py b/polylogue/archive/query/evaluator.py index 88dce272fe..ff76bad30a 100644 --- a/polylogue/archive/query/evaluator.py +++ b/polylogue/archive/query/evaluator.py @@ -132,6 +132,10 @@ def _retained_result( raise RetainedRelationUnavailableError( f"result-set:{result_set_id} has no retained exact relation; use query: to re-evaluate" ) + if manifest.exactness != "exact": + raise RetainedRelationUnavailableError( + f"result-set:{result_set_id} is {manifest.exactness}, not an exact set operand; use query: to re-evaluate" + ) query = get_query(self._conn, manifest.query_hash) lineage = (*extra_lineage, ObjectRef(kind="result-set", object_id=result_set_id)) if query is not None: diff --git a/tests/unit/archive/query/test_evaluator.py b/tests/unit/archive/query/test_evaluator.py index 602e173872..10cb9bc8d4 100644 --- a/tests/unit/archive/query/test_evaluator.py +++ b/tests/unit/archive/query/test_evaluator.py @@ -219,3 +219,32 @@ def test_evaluation_request_rejects_an_unsupported_definition_protocol() -> None with pytest.raises(ValueError, match="unsupported"): QueryEvaluationRequest(query=unsupported, purpose="reference") + + +def test_retained_sampled_result_set_fails_closed_as_a_set_operand() -> None: + conn = _conn() + query = put_query( + conn, + {"field": "title", "value": "sampled"}, + grain="session", + lane="dialogue", + rank_policy="mixed", + created_at_ms=1, + ) + result = put_result_set( + conn, + result_set_id="sampled-retained", + query_hash=query.query_hash, + grain="session", + corpus_epoch="index:g1", + member_refs=("session:sample",), + exactness="sampled", + persistence_class="pinned", + created_at_ms=2, + ) + + with pytest.raises(RetainedRelationUnavailableError, match="not an exact set operand"): + resolve_ref_operand( + RefOperand(ObjectRef(kind="result-set", object_id=result.result_set_id)), + DurableRefResolver(conn, _Evaluator()), + ) From e0acade6ddc8d30953386db409a14214c13a4f8b Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 13 Jul 2026 08:32:53 +0200 Subject: [PATCH 10/10] chore(docs): regenerate topology surfaces after rebase Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_013v95mgEuw3uo9AsnpvhDbm --- docs/plans/topology-target.yaml | 62 ++++++++++++++++----------------- docs/topology-status.md | 6 ++-- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/plans/topology-target.yaml b/docs/plans/topology-target.yaml index f3f000e1bc..b76ef26891 100644 --- a/docs/plans/topology-target.yaml +++ b/docs/plans/topology-target.yaml @@ -218,7 +218,7 @@ files: owner: archive-filter reason: archive-domain filter semantics - path: polylogue/archive/ingest_flags.py - loc: 13 + loc: 15 target: polylogue/archive/ingest_flags.py owner: stable - path: polylogue/archive/message/__init__.py @@ -332,7 +332,7 @@ files: owner: archive-query reason: archive-domain query semantics - path: polylogue/archive/query/evaluator.py - loc: 165 + loc: 173 target: polylogue/archive/query/evaluator.py owner: archive-query reason: archive-domain query semantics @@ -739,11 +739,11 @@ files: target: polylogue/cli/__main__.py owner: stable - path: polylogue/cli/archive_query.py - loc: 2321 + loc: 2443 target: polylogue/cli/archive_query.py owner: stable - path: polylogue/cli/click_app.py - loc: 481 + loc: 571 target: polylogue/cli/click_app.py owner: stable - path: polylogue/cli/click_command_registration.py @@ -751,7 +751,7 @@ files: target: polylogue/cli/click_command_registration.py owner: stable - path: polylogue/cli/click_option_groups.py - loc: 373 + loc: 379 target: polylogue/cli/click_option_groups.py owner: stable - path: polylogue/cli/command_inventory.py @@ -807,11 +807,11 @@ files: target: polylogue/cli/commands/diagnostics.py owner: stable - path: polylogue/cli/commands/embed.py - loc: 811 + loc: 867 target: polylogue/cli/commands/embed.py owner: stable - path: polylogue/cli/commands/facets.py - loc: 63 + loc: 134 target: polylogue/cli/commands/facets.py owner: stable - path: polylogue/cli/commands/hooks.py @@ -835,7 +835,7 @@ files: target: polylogue/cli/commands/judge.py owner: stable - path: polylogue/cli/commands/maintenance.py - loc: 2645 + loc: 2716 target: polylogue/cli/commands/maintenance.py owner: stable - path: polylogue/cli/commands/note.py @@ -871,7 +871,7 @@ files: target: polylogue/cli/convergence_feedback.py owner: stable - path: polylogue/cli/daemon_client.py - loc: 62 + loc: 75 target: polylogue/cli/daemon_client.py owner: stable - path: polylogue/cli/help_markdown.py @@ -935,7 +935,7 @@ files: target: polylogue/cli/query_stats.py owner: stable - path: polylogue/cli/query_verbs.py - loc: 2720 + loc: 2665 target: polylogue/cli/query_verbs.py owner: stable - path: polylogue/cli/read_view_handlers.py @@ -1037,7 +1037,7 @@ files: owner: stable cross_cut: { lifecycle: runtime } - path: polylogue/cli/shared/embed_stats.py - loc: 157 + loc: 178 target: polylogue/cli/shared/embed_stats.py owner: stable - path: polylogue/cli/shared/formatting.py @@ -1229,7 +1229,7 @@ files: owner: core-primitive reason: core primitive - path: polylogue/core/query_identity.py - loc: 168 + loc: 179 target: polylogue/core/query_identity.py owner: core-primitive reason: core primitive @@ -1324,7 +1324,7 @@ files: target: polylogue/daemon/convergence_stages.py owner: stable - path: polylogue/daemon/convergence_standing_queries.py - loc: 302 + loc: 335 target: polylogue/daemon/convergence_standing_queries.py owner: stable - path: polylogue/daemon/cursor_lag_alert.py @@ -1348,7 +1348,7 @@ files: target: polylogue/daemon/embedding_backlog.py owner: stable - path: polylogue/daemon/embedding_readiness.py - loc: 97 + loc: 103 target: polylogue/daemon/embedding_readiness.py owner: stable - path: polylogue/daemon/events.py @@ -1380,7 +1380,7 @@ files: target: polylogue/daemon/healthz.py owner: stable - path: polylogue/daemon/http.py - loc: 4553 + loc: 4609 target: polylogue/daemon/http.py owner: stable - path: polylogue/daemon/lifecycle.py @@ -1449,7 +1449,7 @@ files: target: polylogue/daemon/provenance.py owner: stable - path: polylogue/daemon/route_contracts.py - loc: 598 + loc: 607 target: polylogue/daemon/route_contracts.py owner: stable - path: polylogue/daemon/similarity.py @@ -1473,7 +1473,7 @@ files: target: polylogue/daemon/topology_http.py owner: stable - path: polylogue/daemon/uds.py - loc: 71 + loc: 79 target: polylogue/daemon/uds.py owner: stable - path: polylogue/daemon/user_state_http.py @@ -1908,7 +1908,7 @@ files: target: polylogue/operations/archive.py owner: stable - path: polylogue/operations/archive_debt.py - loc: 1149 + loc: 1155 target: polylogue/operations/archive_debt.py owner: stable - path: polylogue/operations/completion_aggregates.py @@ -2799,7 +2799,7 @@ files: target: polylogue/sources/parsers/beads.py owner: stable - path: polylogue/sources/parsers/browser_capture.py - loc: 335 + loc: 350 target: polylogue/sources/parsers/browser_capture.py owner: stable - path: polylogue/sources/parsers/chatgpt.py @@ -2919,7 +2919,7 @@ files: owner: stable cross_cut: { lifecycle: model } - path: polylogue/sources/revision_backfill.py - loc: 283 + loc: 286 target: polylogue/sources/revision_backfill.py owner: stable - path: polylogue/sources/source_acquisition.py @@ -2927,7 +2927,7 @@ files: target: polylogue/sources/source_acquisition.py owner: stable - path: polylogue/sources/source_acquisition_components.py - loc: 466 + loc: 499 target: polylogue/sources/source_acquisition_components.py owner: stable - path: polylogue/sources/source_parsing.py @@ -3047,7 +3047,7 @@ files: target: polylogue/storage/embeddings/embedding_stats.py owner: stable - path: polylogue/storage/embeddings/materialization.py - loc: 1245 + loc: 1286 target: polylogue/storage/embeddings/materialization.py owner: stable - path: polylogue/storage/embeddings/models.py @@ -3063,7 +3063,7 @@ files: target: polylogue/storage/embeddings/progress.py owner: stable - path: polylogue/storage/embeddings/reconcile.py - loc: 505 + loc: 558 target: polylogue/storage/embeddings/reconcile.py owner: stable - path: polylogue/storage/embeddings/sql.py @@ -3071,7 +3071,7 @@ files: target: polylogue/storage/embeddings/sql.py owner: stable - path: polylogue/storage/embeddings/status_payload.py - loc: 995 + loc: 1159 target: polylogue/storage/embeddings/status_payload.py owner: stable - path: polylogue/storage/embeddings/support.py @@ -3246,7 +3246,7 @@ files: owner: storage-root reason: storage-root cross-cutting helper - path: polylogue/storage/repair.py - loc: 3034 + loc: 4139 target: polylogue/storage/repair.py owner: storage-root reason: storage-root cross-cutting helper @@ -3462,7 +3462,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/__init__.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive.py - loc: 11294 + loc: 11317 target: polylogue/storage/sqlite/archive_tiers/archive.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/archive_init.py @@ -3486,11 +3486,11 @@ files: target: polylogue/storage/sqlite/archive_tiers/context_delivery_write.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/embedding_write.py - loc: 206 + loc: 435 target: polylogue/storage/sqlite/archive_tiers/embedding_write.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/embeddings.py - loc: 36 + loc: 60 target: polylogue/storage/sqlite/archive_tiers/embeddings.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/index.py @@ -3534,7 +3534,7 @@ files: target: polylogue/storage/sqlite/archive_tiers/types.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/user.py - loc: 242 + loc: 290 target: polylogue/storage/sqlite/archive_tiers/user.py owner: stable - path: polylogue/storage/sqlite/archive_tiers/user_annotations.py @@ -3582,7 +3582,7 @@ files: target: polylogue/storage/sqlite/maintenance.py owner: stable - path: polylogue/storage/sqlite/migration_runner.py - loc: 571 + loc: 577 target: polylogue/storage/sqlite/migration_runner.py owner: stable - path: polylogue/storage/sqlite/migrations/__init__.py @@ -3778,7 +3778,7 @@ files: target: polylogue/storage/sqlite/queries/tool_usage.py owner: stable - path: polylogue/storage/sqlite/query_objects.py - loc: 475 + loc: 540 target: polylogue/storage/sqlite/query_objects.py owner: stable - path: polylogue/storage/sqlite/query_store.py diff --git a/docs/topology-status.md b/docs/topology-status.md index 82a637c107..edf28fce99 100644 --- a/docs/topology-status.md +++ b/docs/topology-status.md @@ -28,12 +28,12 @@ Generated by `devtools render topology-status`. Reads `docs/plans/topology-targe ### Summary -- **Stable** (no move scoped): 775 +- **Stable** (no move scoped): 778 - **Kernel** (polylogue/ root): 10 - **Primitives** (storage-root): 18 - **TBD** (cell needs explicit assignment): 6 -- **Total declared**: 929 -- **Realized polylogue/**/*.py**: 929 files declared +- **Total declared**: 932 +- **Realized polylogue/**/*.py**: 932 files declared ### TBD cells (require explicit routing)