From eb5466064933181e56e46254aff1c11ffb10c49e Mon Sep 17 00:00:00 2001 From: Jaixii Date: Thu, 30 Jul 2026 00:29:39 -0400 Subject: [PATCH 01/21] feat(core): add bitemporal token-efficient retrieval --- engraphis/backends/embedder_st.py | 19 +- engraphis/backends/extractor.py | 6 +- engraphis/backends/graph_extractor.py | 31 +- engraphis/backends/sync_folder.py | 10 +- engraphis/backends/vector_numpy.py | 12 +- engraphis/core/consolidate.py | 93 +- engraphis/core/context.py | 468 ++++++ engraphis/core/engine.py | 249 +++- engraphis/core/graphrank.py | 123 +- engraphis/core/grounded.py | 236 ++- engraphis/core/interfaces.py | 82 +- engraphis/core/recall.py | 606 ++++++-- engraphis/core/resolve.py | 72 +- engraphis/core/retrieval_policy.py | 92 ++ engraphis/core/schema.py | 62 +- engraphis/core/store.py | 1733 ++++++++++++++++++---- engraphis/core/sync.py | 61 +- engraphis/logging_setup.py | 11 +- engraphis/mcp_server.py | 221 ++- engraphis/observability.py | 27 + engraphis/read_only_api.py | 58 +- engraphis/routes/v2_api.py | 183 ++- engraphis/service.py | 1965 +++++++++++++++++++++---- tests/test_backends_factories.py | 18 + tests/test_bitemporal_recall.py | 314 ++++ tests/test_canonical_export.py | 456 ++++++ tests/test_code_recall_arm.py | 160 ++ tests/test_compact_recall.py | 212 +++ tests/test_consolidate.py | 53 +- tests/test_context_packing.py | 285 ++++ tests/test_core_store.py | 296 +++- tests/test_engine.py | 233 +++ tests/test_graphrank.py | 135 ++ tests/test_grounded.py | 121 ++ tests/test_mcp_server.py | 160 +- tests/test_read_only_api.py | 50 + tests/test_recall.py | 17 +- tests/test_receipts.py | 345 +++++ tests/test_resolve.py | 124 +- tests/test_retrieval_policy.py | 142 ++ tests/test_service.py | 180 +++ tests/test_service_graph.py | 86 ++ tests/test_store_v4_migration.py | 205 ++- tests/test_sync.py | 140 ++ tests/test_workspace_ops.py | 291 +++- 45 files changed, 9480 insertions(+), 963 deletions(-) create mode 100644 engraphis/core/context.py create mode 100644 engraphis/core/retrieval_policy.py create mode 100644 tests/test_bitemporal_recall.py create mode 100644 tests/test_canonical_export.py create mode 100644 tests/test_code_recall_arm.py create mode 100644 tests/test_compact_recall.py create mode 100644 tests/test_context_packing.py create mode 100644 tests/test_retrieval_policy.py diff --git a/engraphis/backends/embedder_st.py b/engraphis/backends/embedder_st.py index aec30a02..bbb1f643 100644 --- a/engraphis/backends/embedder_st.py +++ b/engraphis/backends/embedder_st.py @@ -15,9 +15,15 @@ class SentenceTransformerEmbedder: - def __init__(self, model_name: str) -> None: + def __init__(self, model_name: str, *, revision: Optional[str] = None) -> None: from sentence_transformers import SentenceTransformer # lazy: optional dependency - self.model = SentenceTransformer(model_name) + kwargs = {"revision": revision} if revision else {} + # Keep declared model provenance beside the loaded object. Benchmark + # artifacts must be able to distinguish a pinned model from a mutable + # fallback without inspecting implementation-specific internals. + self.model_name = model_name + self.revision = revision + self.model = SentenceTransformer(model_name, **kwargs) self._dim = int(self.model.get_embedding_dimension()) @property @@ -34,12 +40,17 @@ def embed(self, texts: list[str], *, kind: Literal["text", "code"] = "text") -> LAST_EMBEDDER_ERROR = "" -def get_embedder(model_name: Optional[str] = None, dim: int = 256): +def get_embedder( + model_name: Optional[str] = None, + dim: int = 256, + *, + revision: Optional[str] = None, +): """A real model if available, else the deterministic offline embedder.""" global LAST_EMBEDDER_ERROR if model_name: try: - emb = SentenceTransformerEmbedder(model_name) + emb = SentenceTransformerEmbedder(model_name, revision=revision) LAST_EMBEDDER_ERROR = "" return emb except Exception as exc: # noqa: BLE001 - optional dep; record why we fall back diff --git a/engraphis/backends/extractor.py b/engraphis/backends/extractor.py index 9cc99ce3..39845302 100644 --- a/engraphis/backends/extractor.py +++ b/engraphis/backends/extractor.py @@ -1,8 +1,8 @@ """Fact extractors — implementations of the ``core.interfaces.Extractor`` protocol. -Modern memory systems often auto-distill raw text into discrete -facts before storage; Engraphis makes that step *pluggable and optional* so the core -stays offline-capable (AGENTS.md §3.8): +Fact extraction can distill raw text into discrete records before storage. Engraphis +makes that step *pluggable and optional* so the core stays offline-capable +(AGENTS.md §3.8): * ``PassthroughExtractor`` — the default: the caller's text is stored exactly as given (today's behaviour, zero dependencies, zero network). diff --git a/engraphis/backends/graph_extractor.py b/engraphis/backends/graph_extractor.py index ba99a351..ac1c7b74 100644 --- a/engraphis/backends/graph_extractor.py +++ b/engraphis/backends/graph_extractor.py @@ -26,7 +26,7 @@ from dataclasses import dataclass, field from typing import Any, Optional -from engraphis.core.interfaces import Edge, Node +from engraphis.core.interfaces import Edge, Node, SearchFilter # ── Regex NER (ported from engraphis/engines/ingest.py — the v1 heuristic path) ── # Capitalized multi-word sequences, emails, hashtags, and mentions. Keep the @@ -346,7 +346,9 @@ def get_graph_extractor(kind: str = "none"): def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] = None, title: str = "", extractor: Any = None, provenance: Optional[dict] = None, commit: bool = True, - extraction: Any = None) -> dict: + extraction: Any = None, + valid_from: Optional[float] = None, + ingested_at: Optional[float] = None) -> dict: """Extract entities/relations from free text and write them into the knowledge graph, scoped to ``(workspace_id, repo_id)``. @@ -383,7 +385,10 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] memory_ids.append(memory_id) prov["memory_ids"] = memory_ids - existing_edges = store.neighbors(list(name_to_id.values())) + existing_edges = store.neighbors( + list(name_to_id.values()), + flt=SearchFilter(workspace_id=workspace_id, repo_id=repo_id), + ) edge_by_key = {(e.src, e.dst, e.relation): e for e in existing_edges} specific_pairs: set[frozenset[str]] = set() written_relations = 0 @@ -397,10 +402,17 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] specific_pairs.add(frozenset((sid, did))) existing = edge_by_key.get(key) if existing is not None: - store.add_edge_support(existing.id, prov, commit=commit) + store.add_edge_support( + existing.id, + prov, + valid_from=valid_from, + ingested_at=ingested_at, + commit=commit, + ) continue eid = store.upsert_edge(Edge(id="", src=sid, dst=did, relation=relation, workspace_id=workspace_id, repo_id=repo_id, + valid_from=valid_from, ingested_at=ingested_at, provenance=prov), commit=commit) edge_by_key[key] = Edge(id=eid, src=sid, dst=did, relation=relation) written_relations += 1 @@ -421,12 +433,19 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] key = (lo, hi, "co_occurs") existing = edge_by_key.get(key) if existing is not None: - store.add_edge_support(existing.id, prov, commit=commit) + store.add_edge_support( + existing.id, + prov, + valid_from=valid_from, + ingested_at=ingested_at, + commit=commit, + ) continue eid = store.upsert_edge(Edge( id="", src=lo, dst=hi, relation="co_occurs", weight=_COOCCUR_WEIGHT, workspace_id=workspace_id, - repo_id=repo_id, provenance=prov, + repo_id=repo_id, valid_from=valid_from, + ingested_at=ingested_at, provenance=prov, ), commit=commit) edge_by_key[key] = Edge(id=eid, src=lo, dst=hi, relation="co_occurs") written_relations += 1 diff --git a/engraphis/backends/sync_folder.py b/engraphis/backends/sync_folder.py index 53ad1152..8bbd9a46 100644 --- a/engraphis/backends/sync_folder.py +++ b/engraphis/backends/sync_folder.py @@ -3,8 +3,8 @@ The zero-infrastructure, self-hostable tier of cloud sync: point two or more devices at the same folder that is *already* replicated between them — a Dropbox / iCloud Drive / OneDrive folder, a Syncthing share, a mounted network drive, or even -a git repo you push/pull — and Engraphis handles the memory-aware merge on top. -This folder transport provides a free local sync path with deterministic bundle merging. +a git repo you push/pull — and Engraphis handles the memory-aware, deterministic +merge on top. It implements the ``SyncTransport`` Protocol (``core/interfaces.py``): opaque named byte blobs, no knowledge of memory semantics. Each device writes exactly one @@ -13,9 +13,9 @@ (temp file + ``os.replace``) so a half-written bundle is never observed — the same mount-safe discipline the rest of the repo uses (AGENTS.md §7). -The managed TLS relay (the headline Pro upsell) is a different ``SyncTransport`` -implementation that plugs in here unchanged. Client-side end-to-end encryption is a -documented follow-up; today's relay stores opaque but plaintext bundle bytes at rest. +The managed TLS relay is a different ``SyncTransport`` implementation that plugs in +here unchanged. Client-side end-to-end encryption is a documented follow-up; today's +relay stores opaque but plaintext bundle bytes at rest. """ from __future__ import annotations diff --git a/engraphis/backends/vector_numpy.py b/engraphis/backends/vector_numpy.py index 7dc8a01b..d28edfd2 100644 --- a/engraphis/backends/vector_numpy.py +++ b/engraphis/backends/vector_numpy.py @@ -30,6 +30,8 @@ def upsert(self, ids: list[str], vecs: np.ndarray, meta: Optional[list[dict]] = def search(self, vec: np.ndarray, k: int, *, filter: Optional[SearchFilter] = None) -> list[tuple[str, float]]: + if k <= 0: + return [] q = np.asarray(vec, dtype=np.float32) n = float(np.linalg.norm(q)) if n > 0: @@ -41,9 +43,13 @@ def search(self, vec: np.ndarray, k: int, mat = np.vstack([r[1] for r in rows]) # already normalized on write scores = mat @ q # cosine == dot for unit vectors k = min(k, len(ids)) - top = np.argpartition(-scores, k - 1)[:k] - top = top[np.argsort(-scores[top])] - return [(ids[i], float(scores[i])) for i in top] + # ``argpartition`` does not define which equal-scored rows survive at + # the top-k boundary. Hashing embeddings produce ties frequently, so + # use the memory id as an explicit stable secondary key. + top = sorted( + range(len(ids)), key=lambda index: (-float(scores[index]), ids[index]) + )[:k] + return [(ids[index], float(scores[index])) for index in top] def delete(self, ids: list[str]) -> None: marks = ",".join("?" for _ in ids) diff --git a/engraphis/core/consolidate.py b/engraphis/core/consolidate.py index 6d9f7e1c..10a41b14 100644 --- a/engraphis/core/consolidate.py +++ b/engraphis/core/consolidate.py @@ -1,7 +1,7 @@ """Sleep-time consolidation (episodic→semantic distillation). -Some systems ship "sleep-time compute" as a cloud service; the local-first equivalent is a -background job the *user* schedules (cron / Windows Task Scheduler / a session hook): +The local-first implementation is a background job the *user* schedules +(cron / Windows Task Scheduler / a session hook): python -m scripts.consolidate --db engraphis.db --workspace acme @@ -127,7 +127,9 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, episodic = store.list_memories( _replace(flt, mtypes=[MemoryType.EPISODIC]), limit=DISTILL_SCAN_LIMIT) - clusters = _cluster_by_subject(episodic, threshold=subject_jaccard) + clusters = _cluster_by_subject( + episodic, threshold=subject_jaccard, store=store, flt=flt, + ) report: dict = {"workspace_id": workspace_id, "repo_id": repo_id, "dry_run": dry_run, "clusters_found": 0, "digests_created": [], "archived": [], @@ -223,12 +225,9 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, store.close_validity( m.id, actor="consolidation", reason=f"retention {r:.4f} below {archive_below} (consolidation sweep)") - try: - engine.index.delete([m.id]) - except Exception as exc: - logger.warning( - "index delete failed for memory %s (%s)", m.id, type(exc).__name__ - ) + # Preserve the vector as historical evidence. Temporal filtering keeps the + # archived row out of current recall while allowing an explicit ``as_of`` + # query to reproduce the semantic result from when it was live. # ── compaction summary: the payoff of the sweep, as a number ───────────── report["compaction"] = { @@ -250,10 +249,71 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None, # ── internals ───────────────────────────────────────────────────────────────── def _cluster_by_subject( - memories: list[MemoryRecord], *, threshold: float + memories: list[MemoryRecord], *, threshold: float, store=None, + flt: Optional[SearchFilter] = None, ) -> list[list[MemoryRecord]]: - """Greedy single-link clustering on token Jaccard — deterministic, order-stable - (memories arrive newest-first from the store; clusters keep that order).""" + """Cluster claim/entity evidence before falling back to token similarity. + + Explicit claim identity is the strongest signal. Persisted memory↔entity + incidence is next, and only records lacking either key take the older + deterministic Jaccard path. Each memory appears in at most one cluster. + """ + keyed: dict[tuple[str, str], list[MemoryRecord]] = {} + assigned: set[str] = set() + for memory in memories: + subject = (memory.subject_key or "").strip() + if subject: + keyed.setdefault((subject, (memory.claim_kind or "").strip()), []).append(memory) + assigned.add(memory.id) + + entity_groups: list[list[MemoryRecord]] = [] + if store is not None: + by_id = {memory.id: memory for memory in memories if memory.id not in assigned} + parent = {memory_id: memory_id for memory_id in by_id} + first_for_entity: dict[str, str] = {} + linked: set[str] = set() + + def find(memory_id: str) -> str: + while parent[memory_id] != memory_id: + parent[memory_id] = parent[parent[memory_id]] + memory_id = parent[memory_id] + return memory_id + + def union(left: str, right: str) -> None: + left_root, right_root = find(left), find(right) + if left_root == right_root: + return + # Stable root selection makes component construction independent of + # incidence-row order and therefore canonical-export friendly. + if left_root > right_root: + left_root, right_root = right_root, left_root + parent[right_root] = left_root + + for link in store.list_memory_entities(flt): + memory = by_id.get(link.get("memory_id")) + entity_id = str(link.get("entity_id") or "") + if memory is None or not entity_id: + continue + linked.add(memory.id) + existing = first_for_entity.setdefault(entity_id, memory.id) + union(existing, memory.id) + + components: dict[str, list[MemoryRecord]] = {} + for memory in memories: + if memory.id in linked: + components.setdefault(find(memory.id), []).append(memory) + assigned.add(memory.id) + entity_groups = list(components.values()) + + remainder = [memory for memory in memories if memory.id not in assigned] + similarity = _cluster_by_similarity(remainder, threshold=threshold) + return [*keyed.values(), *entity_groups, *similarity] + + +def _cluster_by_similarity( + memories: list[MemoryRecord], *, threshold: float, +) -> list[list[MemoryRecord]]: + """Greedy deterministic fallback for memories without durable identity.""" token_sets = [tokenize(f"{m.title} {m.content}") for m in memories] n = len(memories) parent = list(range(n)) @@ -647,14 +707,7 @@ def _write_structured_digests(engine, cluster: list[MemoryRecord], facts: list[d continue engine.store.close_validity( memory.id, at=now, actor="consolidation", reason=reason) - try: - engine.index.delete([memory.id]) - except Exception as exc: - logger.warning( - "index delete failed for memory %s (%s)", - memory.id, - type(exc).__name__, - ) + # Preserve the source vector for historical/as_of retrieval. return ids diff --git a/engraphis/core/context.py b/engraphis/core/context.py new file mode 100644 index 00000000..0e2e8de9 --- /dev/null +++ b/engraphis/core/context.py @@ -0,0 +1,468 @@ +"""Deterministic, token-budgeted context packing. + +The default packer deliberately has no model or tokenizer dependency. It uses a +small, named regex tokenizer so its accounting is exact for the counter it +declares, reproducible offline, and replaceable by benchmark/provider-specific +token counters at the composition boundary. +""" +from __future__ import annotations + +import math +import re +from collections.abc import Callable +from typing import Optional + +from engraphis.core.interfaces import ( + Candidate, + ContextUsage, + PackedChunk, +) + + +_TOKEN_RE = re.compile(r"\w+|[^\w\s]", re.UNICODE) +_SENTENCE_RE = re.compile(r"(?<=[.!?])(?:[\"')\]]*)\s+|\n+") +_WORD_RE = re.compile(r"\w+", re.UNICODE) +_BRIDGE_TERMS = frozenset({ + "call", "calls", "called", "caller", "dependency", "depends", "flow", + "graph", "impact", "path", "related", "relationship", "why", +}) +_QUALIFIER_TERMS = frozenset({ + "cannot", "except", "if", "must", "never", "no", "not", "only", + "unless", "until", "when", "without", +}) + + +class RegexTokenCounter: + """Exact counter for Engraphis' dependency-free tokenization contract.""" + + identity = "engraphis.regex.v1" + + def __call__(self, text: str) -> int: + return len(_TOKEN_RE.findall(text or "")) + + +class DeterministicContextPacker: + """Pack diverse, relevant evidence into a strict token budget. + + Selection is stable for identical inputs. A supersession/consolidation + family contributes at most one member, summaries are preferred when they + retain query evidence, and oversized sources are reduced at sentence + boundaries before a final token-boundary fallback. + """ + + def __init__( + self, + token_counter: Optional[Callable[[str], int]] = None, + *, + token_counter_identity: Optional[str] = None, + ) -> None: + self._count = token_counter or RegexTokenCounter() + self.token_counter_identity = ( + token_counter_identity + or getattr(self._count, "identity", None) + or getattr(self._count, "__name__", None) + or type(self._count).__name__ + ) + + def pack( + self, + query: str, + candidates: list[Candidate], + token_budget: int, + ) -> tuple[str, list[PackedChunk], ContextUsage]: + budget = max(0, int(token_budget)) + source_tokens = sum(self._source_tokens(candidate) for candidate in candidates) + if budget == 0 or not candidates: + return "", [], self._usage( + budget, 0, source_tokens, 0, len(candidates) + ) + + representatives, duplicate_count = _family_representatives(candidates) + query_terms = _terms(query) + needs_bridge = bool(query_terms & _BRIDGE_TERMS) or bool( + re.search(r"(?:\w+[./\\])+\w+|::|->|\b[A-Za-z_]\w*\(\)", query) + ) + ordered = self._selection_order( + representatives, query_terms=query_terms, needs_bridge=needs_bridge + ) + + context = "" + packed: list[PackedChunk] = [] + covered: set[str] = set() + remaining = list(ordered) + + while remaining: + # Re-evaluate novelty after every selection. This gives compact, + # complementary evidence preference over repeated keyword matches. + remaining.sort( + key=lambda candidate: self._utility( + candidate, + query_terms=query_terms, + covered=covered, + needs_bridge=needs_bridge, + ), + reverse=True, + ) + candidate = remaining.pop(0) + record = candidate.record + if record is None: + continue + + prefix = "\n\n" if context else "" + header = self._header(candidate, len(packed) + 1) + base = f"{context}{prefix}{header}\n" + if self._count(base) >= budget: + continue + + available = budget - self._count(base) + excerpt, truncated, reason = self._excerpt( + query, candidate, available + ) + if not excerpt: + continue + proposed = f"{base}{excerpt}" + if self._count(proposed) > budget: + # A custom tokenizer need not be additive. Fit against the + # complete proposed context so the public hard-budget contract + # still holds. + excerpt = self._fit_text( + excerpt, + max_tokens=available, + prefix=base, + total_budget=budget, + ) + truncated = True + reason = "token_boundary_excerpt" + if not excerpt: + continue + proposed = f"{base}{excerpt}" + + context = proposed + packed.append(PackedChunk( + id=candidate.id, + excerpt=excerpt, + tokens=self._count(excerpt), + truncated=truncated, + reason=reason, + )) + covered.update(_terms(excerpt) & query_terms) + + context_tokens = self._count(context) + omitted = len(candidates) - len(packed) + # ``duplicate_count`` is intentionally folded into omitted_count; keep + # the local name to make the family-diversity policy explicit. + omitted = max(omitted, duplicate_count) + return context, packed, self._usage( + budget, context_tokens, source_tokens, len(packed), omitted + ) + + def count_tokens(self, text: str) -> int: + """Count answer text with the exact counter declared by this packer.""" + return int(self._count(text or "")) + + def _selection_order( + self, + candidates: list[Candidate], + *, + query_terms: set[str], + needs_bridge: bool, + ) -> list[Candidate]: + return sorted( + candidates, + key=lambda candidate: self._utility( + candidate, + query_terms=query_terms, + covered=set(), + needs_bridge=needs_bridge, + ), + reverse=True, + ) + + def _utility( + self, + candidate: Candidate, + *, + query_terms: set[str], + covered: set[str], + needs_bridge: bool, + ) -> tuple[float, float, str]: + record = candidate.record + if record is None: + return (-math.inf, -math.inf, candidate.id) + text = f"{record.title} {record.summary or record.content}" + terms = _terms(text) + overlap = terms & query_terms + novelty = len(overlap - covered) / max(1, len(query_terms)) + relevance = max(0.0, float(candidate.score)) + bridge = 0.2 if needs_bridge and candidate.arm in {"graph", "code"} else 0.0 + compactness = 1.0 / math.sqrt(max(1, self._count(text))) + utility = (0.7 * relevance) + (0.25 * novelty) + bridge + (0.05 * compactness) + # Negate the lexical id tie-break while sorting reverse by using a + # stable ordinal derived from the original id separately below. + return (utility, relevance, _reverse_text(candidate.id)) + + def _excerpt( + self, + query: str, + candidate: Candidate, + max_tokens: int, + ) -> tuple[str, bool, str]: + record = candidate.record + if record is None or max_tokens <= 0: + return "", False, "" + full = (record.content or "").strip() + summary = (record.summary or "").strip() + query_terms = _terms(query) + + if summary and self._summary_is_useful(summary, full, query_terms): + if self._count(summary) <= max_tokens: + return summary, summary != full, "summary" + + if full and self._count(full) <= max_tokens: + return full, False, ( + "bridge_evidence" if candidate.arm in {"graph", "code"} else "full" + ) + + excerpt = self._sentence_excerpt(full or summary, query_terms, max_tokens) + if excerpt: + return excerpt, True, ( + "bridge_excerpt" + if candidate.arm in {"graph", "code"} + else "relevant_sentence_excerpt" + ) + fitted = self._fit_text(full or summary, max_tokens=max_tokens) + return fitted, bool(fitted), "token_boundary_excerpt" + + def _summary_is_useful( + self, + summary: str, + full: str, + query_terms: set[str], + ) -> bool: + if not full: + return True + full_overlap = _terms(full) & query_terms + summary_terms = _terms(summary) + preserves_query = not full_overlap or bool(summary_terms & full_overlap) + qualifiers = _terms(full) & _QUALIFIER_TERMS + preserves_qualifiers = qualifiers.issubset(summary_terms) + return preserves_query and preserves_qualifiers + + def _sentence_excerpt( + self, + text: str, + query_terms: set[str], + max_tokens: int, + ) -> str: + sentences = [part.strip() for part in _SENTENCE_RE.split(text) if part.strip()] + if not sentences: + return "" + ranked = sorted( + enumerate(sentences), + key=lambda item: ( + -len(_terms(item[1]) & query_terms), + -len(_terms(item[1]) & _QUALIFIER_TERMS), + item[0], + ), + ) + chosen: list[tuple[int, str]] = [] + qualifier_sentences = [ + item for item in ranked if _terms(item[1]) & _QUALIFIER_TERMS + ] + # A relevant positive sentence without a separate ``unless``/``except``/ + # ``not`` clause can reverse the source's meaning. Admit qualifying + # sentences first; only then spend remaining budget on other evidence. + def admit(items: list[tuple[int, str]]) -> None: + nonlocal chosen + for index, sentence in items: + proposed = " ".join( + value for _, value in sorted(chosen + [(index, sentence)]) + ) + marker = " […]" if len(chosen) + 1 < len(sentences) else "" + if self._count(proposed + marker) <= max_tokens: + chosen.append((index, sentence)) + + admit(qualifier_sentences) + if len(chosen) == len(qualifier_sentences): + admit([item for item in ranked if item not in qualifier_sentences]) + if not chosen: + preferred = qualifier_sentences[0] if qualifier_sentences else ranked[0] + return self._fit_text(preferred[1], max_tokens=max_tokens) + excerpt = " ".join(value for _, value in sorted(chosen)) + if len(chosen) < len(sentences): + marked = f"{excerpt} […]" + if self._count(marked) <= max_tokens: + excerpt = marked + return excerpt + + def _fit_text( + self, + text: str, + *, + max_tokens: int, + prefix: str = "", + total_budget: Optional[int] = None, + ) -> str: + if max_tokens <= 0: + return "" + required_qualifiers = _terms(text) & _QUALIFIER_TERMS + + def semantically_safe(excerpt: str) -> bool: + return required_qualifiers.issubset(_terms(excerpt)) + + tokens = list(_TOKEN_RE.finditer(text)) + if not tokens: + return "" + limit = min(len(tokens), max_tokens) + while limit > 0: + end = tokens[limit - 1].end() + excerpt = text[:end].rstrip() + if limit < len(tokens) and max_tokens > 1: + marked = f"{excerpt} […]" + if self._count(marked) <= max_tokens: + excerpt = marked + within_local = self._count(excerpt) <= max_tokens + within_total = ( + total_budget is None + or self._count(f"{prefix}{excerpt}") <= total_budget + ) + if within_local and within_total and semantically_safe(excerpt): + return excerpt + limit -= 1 + # A custom token counter may split a single regex token (for example a + # character counter or provider tokenizer). In that case there is no + # shorter regex boundary to try, even though a character prefix fits. + # Find the longest safe prefix against the declared counter so tight + # budgets are still used without violating the hard ceiling. + low, high = 1, len(text) + best = "" + while low <= high: + middle = (low + high) // 2 + excerpt = text[:middle].rstrip() + if not excerpt: + low = middle + 1 + continue + marked = f"{excerpt} […]" if middle < len(text) else excerpt + candidate = marked if self._count(marked) <= max_tokens else excerpt + fits = ( + self._count(candidate) <= max_tokens + and ( + total_budget is None + or self._count(f"{prefix}{candidate}") <= total_budget + ) + ) + if fits: + if semantically_safe(candidate): + best = candidate + low = middle + 1 + else: + high = middle - 1 + return best + + def _header(self, candidate: Candidate, ordinal: int) -> str: + record = candidate.record + if record is None: + return f"[{ordinal}]" + # The compact source list carries identity/scope. Repeating ULIDs and + # scope labels inside the context spends reader tokens without adding + # evidence; the ordinal is the citation bridge. + header = f"[{ordinal}]" + if record.title: + title = " ".join(record.title.split())[:120] + header += f" {title}" + return header + + def _source_tokens(self, candidate: Candidate) -> int: + record = candidate.record + if record is None: + return 0 + return self._count(f"{record.title}\n{record.content}") + + def _usage( + self, + budget: int, + context_tokens: int, + source_tokens: int, + packed_count: int, + omitted_count: int, + ) -> ContextUsage: + saved = max(0, source_tokens - context_tokens) + ratio = (saved / source_tokens) if source_tokens else 0.0 + return ContextUsage( + budget_tokens=budget, + context_tokens=context_tokens, + source_tokens=source_tokens, + saved_tokens=saved, + savings_ratio=ratio, + packed_count=packed_count, + omitted_count=max(0, omitted_count), + token_counter=self.token_counter_identity, + ) + + +def _terms(text: str) -> set[str]: + return {match.group(0).casefold() for match in _WORD_RE.finditer(text or "")} + + +def _family_representatives( + candidates: list[Candidate], +) -> tuple[list[Candidate], int]: + """Keep the highest-ranked member of each supersession/consolidation family.""" + parents: dict[str, str] = {} + + def find(value: str) -> str: + parents.setdefault(value, value) + while parents[value] != value: + parents[value] = parents[parents[value]] + value = parents[value] + return value + + def union(left: str, right: str) -> None: + left_root, right_root = find(left), find(right) + if left_root != right_root: + parents[max(left_root, right_root)] = min(left_root, right_root) + + by_claim: dict[str, str] = {} + for candidate in candidates: + find(candidate.id) + record = candidate.record + metadata = record.metadata if record and isinstance(record.metadata, dict) else {} + direct_subject = str(getattr(record, "subject_key", "") or "").strip() + direct_kind = str(getattr(record, "claim_kind", "") or "").strip() + if direct_subject: + claim_identity = f"{direct_subject}\0{direct_kind}" + prior = by_claim.setdefault( + f"subject_key:{claim_identity}", candidate.id + ) + union(candidate.id, prior) + for field in ("subject_key", "claim_key", "consolidation_family"): + value = str(metadata.get(field) or "").strip() + if value: + prior = by_claim.setdefault(f"{field}:{value}", candidate.id) + union(candidate.id, prior) + related = metadata.get("supersedes") or metadata.get("source_ids") or [] + if isinstance(related, str): + related = [related] + if isinstance(related, list): + for item in related: + if isinstance(item, str) and item: + union(candidate.id, item) + + selected: dict[str, Candidate] = {} + for candidate in candidates: + root = find(candidate.id) + current = selected.get(root) + if current is None or (candidate.score, candidate.id) > ( + current.score, + current.id, + ): + selected[root] = candidate + representatives = sorted( + selected.values(), key=lambda candidate: (-candidate.score, candidate.id) + ) + return representatives, len(candidates) - len(representatives) + + +def _reverse_text(value: str) -> str: + # Stable reverse-sort helper without relying on process-randomized hashes. + return "".join(chr(0x10FFFF - ord(char)) for char in value) diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index a3b20def..352c9a50 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -310,6 +310,7 @@ def __init__(self, store: Store, embedder, vector_index, reranker=None, @classmethod def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, + embed_revision: Optional[str] = None, embed_dim: int = 384, vector_backend: str = "auto", rerank_model: Optional[str] = None, extractor: str = "none", graph_extractor: str = "none", @@ -319,7 +320,7 @@ def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, from engraphis.backends.graph_extractor import get_graph_extractor as _get_ge from engraphis.backends.retention import get_retention_supervisor store = Store(db_path, connect=connect) - embedder = get_embedder(embed_model, embed_dim) + embedder = get_embedder(embed_model, embed_dim, revision=embed_revision) index = get_vector_index(store, dim=embedder.dim, prefer=vector_backend) reranker = get_reranker(rerank_model) ext = get_extractor(extractor) @@ -337,7 +338,7 @@ def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = scope: Optional[Scope] = None, title: str = "", importance: float = 0.0, keywords: Optional[list] = None, metadata: Optional[dict] = None, valid_from: Optional[float] = None, resolve_conflicts: bool = True, - candidate_k: int = 5, + candidate_k: int = 5, subject_key: str = "", claim_kind: str = "", _trusted_graph_keys: Optional[frozenset] = None) -> str: """Store one memory. Returns the id of the *live* record: a new id for ADD/ INVALIDATE, or the existing memory's id if this was resolved as a NOOP @@ -347,7 +348,8 @@ def remember(self, content: str, *, workspace_id: str, repo_id: Optional[str] = content, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, mtype=mtype, scope=scope, title=title, importance=importance, keywords=keywords, metadata=metadata, valid_from=valid_from, resolve_conflicts=resolve_conflicts, - candidate_k=candidate_k, _trusted_graph_keys=_trusted_graph_keys, + candidate_k=candidate_k, subject_key=subject_key, claim_kind=claim_kind, + _trusted_graph_keys=_trusted_graph_keys, )["id"] def remember_with_resolution(self, content: str, *, workspace_id: str, @@ -356,6 +358,7 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, title: str = "", importance: float = 0.0, keywords: Optional[list] = None, metadata: Optional[dict] = None, valid_from: Optional[float] = None, resolve_conflicts: bool = True, candidate_k: int = 5, + subject_key: str = "", claim_kind: str = "", _trusted_graph_keys: Optional[frozenset] = None) -> dict: """Store one memory with deterministic conflict resolution. @@ -367,7 +370,20 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, * ``"invalidate"`` — same subject as an existing memory but new content; the old one's validity was closed (never deleted) and this was inserted. ``superseded`` lists the closed id(s). + * ``"relate"`` — evidence shows a nearby claim but not a safe contradiction; + both remain live and a semantic relation is persisted. """ + if valid_from is not None: + if isinstance(valid_from, bool): + raise ValueError("valid_from must be a finite timestamp") + try: + valid_from = float(valid_from) + except (TypeError, ValueError) as exc: + raise ValueError("valid_from must be a finite timestamp") from exc + if not math.isfinite(valid_from): + raise ValueError("valid_from must be a finite timestamp") + subject_key = str(subject_key or "").strip() + claim_kind = str(claim_kind or "").strip() scope_was_omitted = scope is None scope = ( Scope.REPO if (repo_id or session_id) else Scope.WORKSPACE @@ -413,7 +429,8 @@ def remember_with_resolution(self, content: str, *, workspace_id: str, session_id=session_id, mtype=mtype, scope=scope, title=title, importance=importance, keywords=keywords, metadata=metadata, valid_from=valid_from, resolve_conflicts=resolve_conflicts, - candidate_k=candidate_k, trusted_graph_keys=_trusted_graph_keys, + candidate_k=candidate_k, subject_key=subject_key, + claim_kind=claim_kind, trusted_graph_keys=_trusted_graph_keys, ) except BaseException: if (owns_session_transaction @@ -427,6 +444,7 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, title: str, importance: float, keywords: Optional[list], metadata: Optional[dict], valid_from: Optional[float], resolve_conflicts: bool, candidate_k: int, + subject_key: str, claim_kind: str, trusted_graph_keys: Optional[frozenset] = None) -> dict: """The resolve→insert body of ``remember_with_resolution``. The caller holds ``self._write_lock`` for the whole call (atomicity of the resolve decision). @@ -439,8 +457,20 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, decision, neighbors = self._resolve_against_neighbors( text, vec, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, scope=scope, mtype=mtype, - candidate_k=candidate_k, + candidate_k=candidate_k, subject_key=subject_key, + claim_kind=claim_kind, ) + if (decision is not None + and decision.op == ResolutionOp.INVALIDATE + and valid_from is not None): + previous = self.store.get_memory(decision.target_id) + if (previous is not None + and previous.valid_from is not None + and valid_from < previous.valid_from): + raise ValueError( + "valid_from cannot predate the memory it supersedes; " + "record the historical interval separately or correct the older memory" + ) if decision is not None and decision.op == ResolutionOp.NOOP: self.store.reinforce(decision.target_id, boost=scoring.INTERACTION_BOOST["create"]) @@ -469,7 +499,7 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, rec = MemoryRecord( id="", content=content, mtype=mtype, scope=scope, workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, title=title, importance=importance, - stability=stability, + stability=stability, subject_key=subject_key, claim_kind=claim_kind, keywords=keywords or [], metadata=meta, valid_from=valid_from, # Lift provenance into its dedicated field/column so recall/why/timeline # surface it (copied, not popped: consolidate.py still reads @@ -518,7 +548,8 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, _graph_feed(self.store, content, workspace_id=workspace_id, repo_id=repo_id, title=title, extractor=StructuredMetadataGraphExtractor(meta), - provenance={"source": "structured_extractor", "memory_id": mid}) + provenance={"source": "structured_extractor", "memory_id": mid}, + valid_from=rec.valid_from, ingested_at=rec.ingested_at) except Exception: pass if scope != Scope.SESSION and self.graph_extractor is not None: @@ -526,18 +557,27 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, from engraphis.backends.graph_extractor import feed as _graph_feed _graph_feed(self.store, content, workspace_id=workspace_id, repo_id=repo_id, title=title, extractor=self.graph_extractor, - provenance={"source": "graph_extractor", "memory_id": mid}) + provenance={"source": "graph_extractor", "memory_id": mid}, + valid_from=rec.valid_from, ingested_at=rec.ingested_at) except Exception: pass + if scope != Scope.SESSION: + self._link_memory_entities( + mid, content, workspace_id=workspace_id, repo_id=repo_id, + valid_from=rec.valid_from, + ) if decision is not None and decision.op == ResolutionOp.INVALIDATE: - self.store.close_validity(decision.target_id, reason=decision.reason) - try: - self.index.delete([decision.target_id]) - except Exception as exc: # noqa: BLE001 — merely stale in the index; recall - # re-checks validity on read, so log (don't audit) and continue. - logger.warning("vector-index delete failed for %s (%s)", - decision.target_id, type(exc).__name__) + # World time closes when the replacement becomes true, not when this process + # happened to ingest it. This keeps backdated and scheduled facts queryable at + # the correct ``as_of`` anchor. + self.store.close_validity( + decision.target_id, at=rec.valid_from, reason=decision.reason + ) + # Keep the superseded vector. Every vector backend applies the same temporal + # SearchFilter as lexical/graph retrieval, so it is hidden from current recall + # but remains available for historical ``as_of`` queries. Deleting it made + # time travel silently lose the semantic arm. self.store.audit("resolver", "invalidate", decision.target_id, decision.reason) linked = self._evolve(mid, neighbors, exclude={decision.target_id}) out = {"id": mid, "op": "invalidate", "superseded": [decision.target_id], @@ -547,7 +587,16 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, return out linked = self._evolve(mid, neighbors) - out = {"id": mid, "op": "add", "reason": decision.reason if decision else ""} + if decision is not None and decision.op == ResolutionOp.RELATE: + related_to = decision.target_id + if related_to and not self.store.has_link(mid, related_to): + self.store.add_link(mid, related_to, "related", reason=decision.reason) + out = { + "id": mid, "op": "relate", "related_to": related_to, + "reason": decision.reason, + } + else: + out = {"id": mid, "op": "add", "reason": decision.reason if decision else ""} if linked: out["linked"] = linked return out @@ -630,6 +679,36 @@ def _has_structured_graph_metadata(self, metadata: dict) -> bool: or isinstance(structured.get("relations"), list) ) + def _link_memory_entities(self, memory_id: str, content: str, *, + workspace_id: str, repo_id: Optional[str], + valid_from: Optional[float]) -> None: + """Persist edge-derived and exact textual entity evidence for one memory.""" + owns_transaction = not self.store.conn.transaction_owned_by_current_thread() + try: + self.store.backfill_memory_entities_for_memory(memory_id) + entities = self.store.list_entities(SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, + )) + for entity in entities: + name = (entity.name or "").strip() + if len(name) < 2: + continue + if re.search(r"(? list[str]: """A-MEM-style memory evolution on write: a new memory auto-links to its closest still-live neighbors and gives them a small @@ -663,7 +742,8 @@ def _evolve(self, new_id: str, neighbors: list, *, exclude: Optional[set] = None def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id: str, repo_id: Optional[str], session_id: Optional[str], - scope: Scope, mtype: MemoryType, candidate_k: int): + scope: Scope, mtype: MemoryType, candidate_k: int, + subject_key: str = "", claim_kind: str = ""): """Fetch same-scope neighbors via the vector index and run the deterministic resolver (``core.resolve``). Returns ``(decision, neighbors)`` so the caller can also evolve the neighborhood. Never raises — a broken/missing index degrades to @@ -687,7 +767,9 @@ def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id and nrec.expired_at is None and (nrec.valid_to is None or nrec.valid_to > now)): neighbors.append((sim, nrec)) - return resolve(text, neighbors), neighbors + return resolve( + text, neighbors, subject_key=subject_key, claim_kind=claim_kind, + ), neighbors # ── ingest: extract-then-remember ─────────────────────────────────────────── def ingest(self, text: str, *, workspace_id: str, repo_id: Optional[str] = None, @@ -767,7 +849,9 @@ def consolidate(self, *, workspace_id: str, repo_id: Optional[str] = None, # ── read ────────────────────────────────────────────────────────────────── def _recall_filter(self, *, workspace_id: Optional[str], repo_id: Optional[str], session_id: Optional[str], scopes: Optional[list], - mtypes: Optional[list], as_of: Optional[float]) -> SearchFilter: + mtypes: Optional[list], as_of: Optional[float], + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> SearchFilter: """Build an ancestor-aware filter, resolving a session's parent repo in core. The service performs the same validation for friendly error payloads, but direct @@ -785,25 +869,40 @@ def _recall_filter(self, *, workspace_id: Optional[str], repo_id: Optional[str], repo_id = repo_id or session.get("repo_id") return SearchFilter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, - scopes=scopes, mtypes=mtypes, as_of=as_of, include_ancestors=True, + scopes=scopes, mtypes=mtypes, as_of=as_of, valid_at=valid_at, + known_at=known_at, include_ancestors=True, ) def recall(self, query: str, *, workspace_id: Optional[str] = None, repo_id: Optional[str] = None, session_id: Optional[str] = None, scopes: Optional[list] = None, mtypes: Optional[list] = None, as_of: Optional[float] = None, - k: int = 8) -> RecallResult: + valid_at: Optional[float] = None, known_at: Optional[float] = None, + k: int = 8, token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", diagnostics: bool = False, + reinforce: bool = False) -> RecallResult: flt = self._recall_filter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, - scopes=scopes, mtypes=mtypes, as_of=as_of, + scopes=scopes, mtypes=mtypes, as_of=as_of, valid_at=valid_at, + known_at=known_at, + ) + # Recall is observational unless the caller has an explicit use signal. + # Historical inspection is always observational: reinforcement would make a + # past reconstruction alter future ranking. + return self.recall_engine.recall( + query, flt, k=k, reinforce=bool(reinforce) and not flt.historical, + token_budget=token_budget, retrieval_profile=retrieval_profile, + diagnostics=diagnostics, ) - return self.recall_engine.recall(query, flt, k=k) def grounded_recall(self, query: str, *, workspace_id: Optional[str] = None, repo_id: Optional[str] = None, session_id: Optional[str] = None, scopes: Optional[list] = None, mtypes: Optional[list] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, known_at: Optional[float] = None, k: int = 8, llm=None, min_support: Optional[float] = None, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", diagnostics: bool = False, max_citations: int = 5, reinforce: bool = True): """Recall, then answer *strictly from* what was recalled — with citations and an explicit abstain when the evidence is too weak (``core.grounded``). Offline and @@ -817,16 +916,20 @@ def grounded_recall(self, query: str, *, workspace_id: Optional[str] = None, from engraphis.core import grounded as _grounded flt = self._recall_filter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, - scopes=scopes, mtypes=mtypes, as_of=as_of, + scopes=scopes, mtypes=mtypes, as_of=as_of, valid_at=valid_at, + known_at=known_at, ) # Recall without reinforcing here: a grounded read should reward only the memories # it actually cites, and an abstain should reward nothing — don't reinforce the # irrelevant nearest-neighbours an off-topic query happened to surface. - result = self.recall_engine.recall(query, flt, k=k, reinforce=False) + result = self.recall_engine.recall( + query, flt, k=k, reinforce=False, token_budget=token_budget, + retrieval_profile=retrieval_profile, diagnostics=diagnostics, + ) floor = _grounded.GROUNDED_SUPPORT_FLOOR if min_support is None else min_support answer = _grounded.build_grounded_answer(query, result, self.embedder, llm=llm, min_support=floor, max_citations=max_citations) - if reinforce and answer.grounded: + if reinforce and not flt.historical and answer.grounded: for cite in answer.citations: if cite.get("id"): self.store.reinforce(cite["id"], boost=scoring.INTERACTION_BOOST["recall"]) @@ -931,10 +1034,8 @@ def forget(self, memory_id: str, *, reason: str = "", actor: str = "user") -> di if self.store.get_memory(memory_id) is None: raise KeyError(f"no memory with id '{memory_id}'") self.store.close_validity(memory_id, actor=actor, reason=reason or "forgotten by request") - try: - self.index.delete([memory_id]) - except Exception: - pass + # Preserve the vector for explicit historical/as_of recall. Temporal filtering + # keeps this retired row out of the current live view. return {"id": memory_id, "status": "forgotten", "reason": reason} def pin(self, memory_id: str, *, pinned: bool = True, actor: str = "user") -> dict: @@ -980,10 +1081,8 @@ def correct(self, memory_id: str, new_content: str, *, reason: str = "", if old.pinned: self.store.set_pinned(new_id, True) self.store.close_validity(memory_id, actor=actor, reason=reason or "corrected") - try: - self.index.delete([memory_id]) - except Exception: - pass + # The old vector is historical evidence; SearchFilter validity hides it from + # current recall while keeping semantic time travel complete. return {"id": new_id, "superseded": [memory_id], "reason": reason} def promote(self, memory_id: str, target_scope: Scope, *, reason: str = "", @@ -1096,10 +1195,7 @@ def promote(self, memory_id: str, target_scope: Scope, *, reason: str = "", old.id, actor=actor, reason=reason or f"promoted from {old.scope.value} to {target_scope.value}", ) - try: - self.index.delete([old.id]) - except Exception: - pass + # Preserve the source vector for historical/as_of inspection. if not self.store.has_link(promoted_id, old.id, relation="promotes"): self.store.add_link( promoted_id, old.id, "promotes", reason=reason or "scope promotion" @@ -1207,10 +1303,7 @@ def merge(self, source_ids: list, merged_content: str, *, for r in sources: self.store.close_validity(r.id, actor=actor, reason=reason or "merged into a combined memory") - try: - self.index.delete([r.id]) - except Exception: - pass + # Preserve source vectors for historical/as_of retrieval. # Linking/auditing stays a separate pass so the audit trail keeps its original # shape: every source's invalidate entry, then every source's merge entry. for r in sources: @@ -1434,9 +1527,12 @@ def search_code(self, query: str, *, repo_id: str, limit: int = 20, """Symbol-graph + lexical code search — far cheaper than dumping files for structural questions, and (via ``called_by``) answers "what breaks if I change X" directly from the call graph.""" - symbols = self.store.search_symbols(repo_id, query, limit=limit) + self._validate_code_filter(repo_id, flt) + symbols = self.store.search_symbols(repo_id, query, limit=limit, flt=flt) for s in symbols: - s["called_by"] = self.store.get_symbol_callers(repo_id, s["name"], limit=10) + s["called_by"] = self.store.get_symbol_callers( + repo_id, s["name"], limit=10, flt=flt + ) s["linked_memories"] = self.store.memories_for_symbol( repo_id, s["id"], flt=flt, limit=10 ) @@ -1523,10 +1619,6 @@ def rebuild_code_memory_links(self, *, repo_id: str) -> int: ) if not records: break - memory_ids = [record.id for record in records] - self.store.clear_code_memory_links_for_memories( - repo_id, memory_ids, commit=False, - ) linked_per_memory = {record.id: 0 for record in records} symbol_cursor: Optional[tuple[str, str, str]] = None while True: @@ -1562,8 +1654,9 @@ def rebuild_code_memory_links(self, *, repo_id: str) -> int: def code_path(self, source: str, target: str, *, repo_id: str, max_depth: int = 8, flt: Optional[SearchFilter] = None) -> dict: """Shortest path across definitions, calls, imports, and symbol aliases.""" - symbols = self.store.list_symbols(repo_id) - stored_edges = self.store.list_code_edges(repo_id) + self._validate_code_filter(repo_id, flt) + symbols = self.store.list_symbols(repo_id, flt=flt) + stored_edges = self.store.list_code_edges(repo_id, flt=flt) adjacency: dict[str, list[tuple[str, dict, bool]]] = defaultdict(list) node_meta: dict[str, dict] = {} for sym in symbols: @@ -1586,13 +1679,7 @@ def code_path(self, source: str, target: str, *, repo_id: str, node_meta.setdefault(src, {"kind": "code", "name": src}) node_meta.setdefault(dst, {"kind": "code", "name": dst}) symbol_by_id = {symbol["id"]: symbol for symbol in symbols} - now = now_ts() for link in self.store.list_code_memory_links(repo_id, flt=flt): - if link.get("expired_at") is not None: - continue - valid_to = link.get("valid_to") - if valid_to is not None and now >= float(valid_to): - continue symbol = symbol_by_id.get(link.get("symbol_id")) if not symbol or not link.get("memory_id"): continue @@ -1705,15 +1792,17 @@ def _resolve_code_node(query: str, symbols: list[dict], def analyze_code_graph(self, *, repo_id: str, limit: Optional[int] = None, - edge_limit: Optional[int] = None) -> dict: + edge_limit: Optional[int] = None, + flt: Optional[SearchFilter] = None) -> dict: """Deterministic weighted communities, hotspots, and cross-file connections. ``limit``/``edge_limit`` bound the symbol/edge fetch. They default to ``None`` (unbounded) so ``analyze_impact`` keeps today's exact answer; ``export_code_graph`` passes its own caps because that payload is reachable by a ``viewer``. """ - edges = self.store.list_code_edges(repo_id, limit=edge_limit) - symbols = self.store.list_symbols(repo_id, limit=limit) + self._validate_code_filter(repo_id, flt) + edges = self.store.list_code_edges(repo_id, limit=edge_limit, flt=flt) + symbols = self.store.list_symbols(repo_id, limit=limit, flt=flt) adjacency: dict[str, dict[str, float]] = defaultdict(dict) degree: dict[str, int] = defaultdict(int) for edge in edges: @@ -1814,6 +1903,7 @@ def analyze_code_graph(self, *, repo_id: str, def analyze_impact(self, changed_files: list[str], *, repo_id: str, flt: Optional[SearchFilter] = None) -> dict: """Estimate graph and memory impact for a git diff / PR file list.""" + self._validate_code_filter(repo_id, flt) normalized = [] seen = set() for file in changed_files: @@ -1825,12 +1915,12 @@ def analyze_impact(self, changed_files: list[str], *, repo_id: str, if rel and rel not in seen: seen.add(rel) normalized.append(rel) - symbols = self.store.symbols_for_files(repo_id, normalized) + symbols = self.store.symbols_for_files(repo_id, normalized, flt=flt) touched_names = { name for sym in symbols for name in (sym.get("name"), sym.get("fqname")) if name } touched_leaf_names = {str(name).split(".")[-1] for name in touched_names} - edges = self.store.list_code_edges(repo_id) + edges = self.store.list_code_edges(repo_id, flt=flt) inbound = [ edge for edge in edges if edge.get("dst") in touched_names @@ -1844,13 +1934,7 @@ def analyze_impact(self, changed_files: list[str], *, repo_id: str, memory_mentions: dict[str, dict] = {} touched_symbol_ids = {symbol["id"] for symbol in symbols} - now = now_ts() for link in self.store.list_code_memory_links(repo_id, flt=flt): - if link.get("expired_at") is not None: - continue - valid_to = link.get("valid_to") - if valid_to is not None and now >= float(valid_to): - continue if link.get("symbol_id") not in touched_symbol_ids: continue item = memory_mentions.setdefault( @@ -1881,7 +1965,7 @@ def analyze_impact(self, changed_files: list[str], *, repo_id: str, ) item["symbols"].append(name) - analysis = self.analyze_code_graph(repo_id=repo_id) + analysis = self.analyze_code_graph(repo_id=repo_id, flt=flt) node_community = analysis.pop("_node_community") communities_affected = sorted({ node_community[name] for name in touched_names if name in node_community @@ -1940,16 +2024,17 @@ def export_code_graph(self, *, repo_id: str, """ limit = max(1, min(CODE_EXPORT_MAX_LIMIT, int(limit))) edge_cap = max(limit * 8, 2_000) + self._validate_code_filter(repo_id, flt) analysis = self.analyze_code_graph(repo_id=repo_id, limit=limit, - edge_limit=edge_cap) + edge_limit=edge_cap, flt=flt) analysis.pop("_node_community", None) # Fetch one sentinel row beyond the payload cap so truncation stays observable # without materializing every indexed file in a large repository. files = self.store.list_code_files(repo_id, limit=limit + 1) truncated_files = len(files) > limit files = files[:limit] - nodes = self.store.list_symbols(repo_id, limit=limit) - edges = self.store.list_code_edges(repo_id, limit=edge_cap) + nodes = self.store.list_symbols(repo_id, limit=limit, flt=flt) + edges = self.store.list_code_edges(repo_id, limit=edge_cap, flt=flt) memory_links = self.store.list_code_memory_links( repo_id, flt=flt, limit=edge_cap ) @@ -1970,6 +2055,28 @@ def export_code_graph(self, *, repo_id: str, "analysis": analysis, } + def _validate_code_filter( + self, repo_id: str, flt: Optional[SearchFilter] + ) -> None: + """Reject inconsistent repo/workspace filters before any code row is read. + + Code-history tables are keyed by ``repo_id`` rather than duplicating a + workspace column. Without this check, a direct engine caller could pair a + workspace-A filter with a workspace-B repo id and receive B's symbols even + though memory reads correctly returned nothing. + """ + if flt is None: + return + if flt.repo_id is not None and flt.repo_id != repo_id: + raise ValueError("code filter repo_id does not match the requested repo") + if flt.workspace_id is None: + return + row = self.store.conn.execute( + "SELECT workspace_id FROM repos WHERE id=?", (repo_id,) + ).fetchone() + if row is None or row["workspace_id"] != flt.workspace_id: + raise ValueError("code filter workspace_id does not own the requested repo") + def code_graph_report(self, *, repo_id: str, payload: Optional[dict] = None, flt: Optional[SearchFilter] = None) -> str: """Human-readable GRAPH_REPORT.md companion to :meth:`export_code_graph`. diff --git a/engraphis/core/graphrank.py b/engraphis/core/graphrank.py index 34d73325..38a1e744 100644 --- a/engraphis/core/graphrank.py +++ b/engraphis/core/graphrank.py @@ -1,20 +1,24 @@ -"""Personalized PageRank over the memory/entity graph. +"""Deterministic sparse personalized PageRank for local memory graphs. -HippoRAG-style single-step graph retrieval: seed the walk at the query's entities and -let the stationary distribution rank everything reachable — multi-hop associations -included — instead of expanding a fixed number of hops. Pure NumPy (AGENTS.md §3.8), -deterministic, and sized for the local-first reality: the adjacency is built per query -from the scoped store (hundreds to low thousands of nodes), where a dense power -iteration is both exact and fast. A sparse/persistent implementation can replace this -behind the same function signature when scale demands it. +The graph arm can contain thousands of entities, memories, and links. A dense +``N × N`` transition matrix turns an otherwise modest local graph into quadratic +memory pressure, so this implementation stores only normalized outgoing edges +and walks them directly. It deliberately depends on no sparse-matrix package. """ from __future__ import annotations -import numpy as np +import math + DAMPING = 0.85 ITERATIONS = 30 TOL = 1e-9 +# Safety limits for direct callers. Recall already builds a bounded scoped graph; +# these make a malformed local/plugin adjacency fail deterministically rather than +# allocating unbounded state. They are comfortably above normal local graph arms. +MAX_NODES = 100_000 +MAX_EDGES = 1_000_000 +MAX_ITERATIONS = 100 def personalized_pagerank( @@ -25,49 +29,82 @@ def personalized_pagerank( iterations: int = ITERATIONS, tol: float = TOL, ) -> dict[str, float]: - """Rank nodes by their stationary probability under a random walk with restart. + """Rank nodes by a sparse random walk with restart. - ``adjacency`` maps node -> [(neighbor, weight), ...]; pass both directions for an - undirected graph. ``seeds`` are the restart set (unknown seeds are ignored). Nodes - unreachable from every seed score 0. Returns {} when there is nothing to walk. + ``adjacency`` maps node -> ``[(neighbor, weight), ...]``; pass both + directions for an undirected graph. Unknown seed ids retain the legacy + restart behavior when at least one seed has outgoing adjacency. Oversized + inputs return ``{}`` deterministically instead of attempting an unbounded + local computation. """ if not adjacency or not seeds: return {} - nodes: list[str] = sorted( - set(adjacency) - | {dst for nbrs in adjacency.values() for dst, _ in nbrs} - | set(seeds) - ) - idx = {n: i for i, n in enumerate(nodes)} - n = len(nodes) + nodes = set(adjacency) + edge_count = 0 + for neighbors in adjacency.values(): + edge_count += len(neighbors) + if edge_count > MAX_EDGES: + return {} + nodes.update(dst for dst, _ in neighbors) + nodes.update(seeds) + if len(nodes) > MAX_NODES: + return {} - seed_ids = [idx[s] for s in seeds if s in idx] - live_seeds = [s for s in seeds if s in adjacency and adjacency[s]] + ordered_nodes = sorted(nodes) + node_index = {node: index for index, node in enumerate(ordered_nodes)} + n_nodes = len(ordered_nodes) + seed_ids = [node_index[seed] for seed in seeds if seed in node_index] + live_seeds = [seed for seed in seeds if seed in adjacency and adjacency[seed]] if not seed_ids or not live_seeds: return {} - # Column-stochastic transition matrix; dangling nodes restart to the seeds. - M = np.zeros((n, n), dtype=np.float64) - for src, nbrs in adjacency.items(): - col = idx[src] - total = float(sum(max(w, 0.0) for _, w in nbrs)) - if total <= 0.0: + # Aggregate duplicate destinations before applying a source's mass. This + # matches the old dense matrix's ``M[dst, src] += ...`` semantics while + # keeping the storage and each iteration O(nodes + edges). + outgoing: list[list[tuple[int, float]]] = [[] for _ in range(n_nodes)] + for source in ordered_nodes: + neighbors = adjacency.get(source, []) + total = sum(max(float(weight), 0.0) for _, weight in neighbors) + if total <= 0.0 or not math.isfinite(total): continue - for dst, w in nbrs: - if w > 0.0: - M[idx[dst], col] += w / total - - restart = np.zeros(n, dtype=np.float64) - restart[seed_ids] = 1.0 / len(seed_ids) - dangling = M.sum(axis=0) == 0.0 + destination_weights: dict[int, float] = {} + for destination, weight in neighbors: + if weight > 0.0: + destination_id = node_index[destination] + destination_weights[destination_id] = ( + destination_weights.get(destination_id, 0.0) + float(weight) / total + ) + outgoing[node_index[source]] = list(destination_weights.items()) - p = restart.copy() - for _ in range(iterations): - spread = M @ p + p[dangling].sum() * restart - p_next = (1.0 - damping) * restart + damping * spread - if float(np.abs(p_next - p).sum()) < tol: - p = p_next + restart = [0.0] * n_nodes + for seed_id in seed_ids: + restart[seed_id] = 1.0 / len(seed_ids) + dangling = [index for index, neighbors in enumerate(outgoing) if not neighbors] + probability = restart[:] + iteration_limit = max(0, min(int(iterations), MAX_ITERATIONS)) + for _ in range(iteration_limit): + spread = [0.0] * n_nodes + for source_id, edges in enumerate(outgoing): + if probability[source_id] == 0.0: + continue + for destination_id, weight in edges: + spread[destination_id] += probability[source_id] * weight + dangling_mass = sum(probability[index] for index in dangling) + if dangling_mass: + for index, weight in enumerate(restart): + if weight: + spread[index] += dangling_mass * weight + next_probability = [ + (1.0 - damping) * restart[index] + damping * spread[index] + for index in range(n_nodes) + ] + if sum(abs(after - before) for after, before in zip(next_probability, probability)) < tol: + probability = next_probability break - p = p_next + probability = next_probability - return {nodes[i]: float(p[i]) for i in range(n) if p[i] > 0.0} + return { + ordered_nodes[index]: score + for index, score in enumerate(probability) + if score > 0.0 + } diff --git a/engraphis/core/grounded.py b/engraphis/core/grounded.py index b49a8664..179c3b59 100644 --- a/engraphis/core/grounded.py +++ b/engraphis/core/grounded.py @@ -11,10 +11,10 @@ * **Deterministic (offline default).** No LLM. The answer is an *extractive* stitch of the cited memories — it never introduces a claim that is not in a source. The - groundedness verdict is computed from an absolute query-memory support signal (the - max of semantic cosine and lexical Jaccard), independent of the relative, per-query - recall score, so "insufficient evidence" is a real threshold rather than a ranking - artefact. + groundedness verdict is computed from an absolute query-memory support signal + (semantic cosine plus lexical/predicate agreement), independent of the relative, + per-query recall score, so "insufficient evidence" is a real threshold rather than + a ranking artefact. * **Synthesised (opt-in).** If an object implementing ``core.interfaces.LLM`` is injected, it may write prose — but constrained to the same numbered sources and the same abstain sentinel, and it degrades to the extractive answer on any error. @@ -27,12 +27,14 @@ """ from __future__ import annotations +import math import re -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from typing import Optional import numpy as np +from engraphis.core.context import RegexTokenCounter from engraphis.core.interfaces import LLM from engraphis.core.recall import RecallResult from engraphis.core.textutil import jaccard, tokenize @@ -46,6 +48,16 @@ GROUNDED_SUPPORT_FLOOR = 0.25 ABSTAIN_SENTINEL = "INSUFFICIENT_EVIDENCE" _CITE_RE = re.compile(r"\[(\d+)\]") +_QUERY_FRAMING_TERMS = { + "what", "which", "who", "where", "when", "why", "how", "scheme", "format", +} +# Words which can make a citation grammatical without making an additional factual +# claim. The LLM verifier below deliberately permits only these words in addition +# to source tokens. Unknown paraphrases safely fall back to extractive evidence. +_SYNTHESIS_GLUE_TERMS = { + "according", "answer", "answers", "based", "evidence", "indicates", "per", + "provided", "said", "says", "source", "sources", "states", "supports", +} @dataclass @@ -63,9 +75,16 @@ class GroundedAnswer: support: float = 0.0 synthesized: bool = False citations: list[dict] = field(default_factory=list) + usage: dict = field(default_factory=dict) + packed_sources: list[dict] = field(default_factory=list) + valid_at: Optional[float] = None + known_at: Optional[float] = None + historical: bool = False + retrieval_profile: str = "balanced" + retrieval_trace: Optional[list[dict]] = None def to_dict(self) -> dict: - return { + payload = { "answer": self.answer, "grounded": self.grounded, "abstained": self.abstained, @@ -73,7 +92,16 @@ def to_dict(self) -> dict: "support": round(self.support, 4), "synthesized": self.synthesized, "citations": self.citations, + "usage": self.usage, + "packed_sources": self.packed_sources, + "valid_at": self.valid_at, + "known_at": self.known_at, + "historical": self.historical, + "retrieval_profile": self.retrieval_profile, } + if self.retrieval_trace is not None: + payload["retrieval_trace"] = self.retrieval_trace + return payload def _filtered_text(text: str) -> str: @@ -85,37 +113,129 @@ def _filtered_text(text: str) -> str: return " ".join(sorted(toks)) if toks else (text or "") +def _related_term_count(query_tokens: set[str], content_tokens: set[str]) -> int: + """Count conservative exact/morphological term matches. + + A single shared topic word is not evidence for the query's predicate + (``bake sourdough`` versus ``orders sourdough``). Prefix agreement also + recognizes ordinary inflections such as ``token``/``tokens`` and + ``standardise``/``standardised`` without a language model. + """ + matched = 0 + for query_term in query_tokens: + for content_term in content_tokens: + if query_term == content_term: + matched += 1 + break + shorter = min(len(query_term), len(content_term)) + if shorter < 5: + continue + common = 0 + for left, right in zip(query_term, content_term): + if left != right: + break + common += 1 + if common >= max(5, min(7, shorter)): + matched += 1 + break + return matched + + def _support_scores(query: str, contents: list[str], embedder) -> list[float]: - """Absolute per-source support = max(semantic cosine, lexical Jaccard), in [0, 1]. + """Absolute per-source support from semantic, lexical, and predicate agreement. Both arms are query-independent in scale — unlike the recall score, which is min-max normalised *per query* and so cannot be compared against a fixed threshold. That is why groundedness is recomputed here rather than read off ``chunk["score"]``. The - cosine is taken over *stopword-filtered* text so shared filler words don't register - as evidence. + cosine is taken over *stopword-filtered* text, then conservatively discounted when + a multi-term query and source share only one topic term. """ if not contents: return [] - q_tokens = tokenize(query) + q_tokens = tokenize(query) - _QUERY_FRAMING_TERMS texts = [_filtered_text(query)] + [_filtered_text(c) for c in contents] vecs = embedder.embed(texts) qn = np.asarray(vecs[0], dtype=float) qn = qn / (float(np.linalg.norm(qn)) or 1.0) out: list[float] = [] for i, content in enumerate(contents): + content_tokens = tokenize(content) cv = np.asarray(vecs[i + 1], dtype=float) cn = cv / (float(np.linalg.norm(cv)) or 1.0) - cos = float(np.dot(qn, cn)) - lex = jaccard(q_tokens, tokenize(content)) + cos = max(0.0, float(np.dot(qn, cn))) + lex = jaccard(q_tokens, content_tokens) + related_terms = _related_term_count(q_tokens, content_tokens) + # Hashing and dense embedders can consider two texts topically similar + # when they share one salient noun but make unrelated claims. Require a + # second predicate/qualifier match for ordinary multi-term questions, + # while allowing genuinely strong semantic paraphrases to stand alone. + if len(q_tokens) >= 3 and related_terms < 2 and cos < 0.6: + cos *= related_terms / 2.0 out.append(max(cos, lex)) return out -def _cites_a_source(text: str, n_citations: int) -> bool: - """True if ``text`` has at least one ``[i]`` marker with ``1 <= i <= n_citations``. - Guards the synthesised path: prose that cites nothing may have introduced an uncited - (possibly fabricated) claim, so it is rejected in favour of the extractive answer.""" - return any(1 <= int(m) <= n_citations for m in _CITE_RE.findall(text)) +def _citations_are_valid(text: str, n_citations: int) -> bool: + """Require at least one citation and reject every out-of-range marker. + + Accepting prose merely because *one* marker was valid let an answer combine + ``[1]`` with fabricated ``[99]`` evidence. Structural citation integrity is + fail-closed: every numbered source reference must resolve to a supplied source. + """ + markers = [int(marker) for marker in _CITE_RE.findall(text)] + return bool(markers) and all(1 <= marker <= n_citations for marker in markers) + + +def _ordered_tokens(text: str) -> list[str]: + """Case-folded lexical tokens with order and small numbers preserved.""" + return re.findall(r"[^\W_]+", text.casefold(), flags=re.UNICODE) + + +def _contains_span(source: list[str], claim: list[str]) -> bool: + """Whether ``claim`` is one exact contiguous lexical span of ``source``.""" + width = len(claim) + return bool(width) and any( + source[start:start + width] == claim + for start in range(0, len(source) - width + 1) + ) + + +def _synthesis_is_source_bounded(text: str, citations: list[dict]) -> bool: + """Return whether each cited synthesis clause is extractive from one source. + + Citation syntax alone cannot prove a generated claim is present in its source: + ``Invented fact [1]`` has a valid marker but no evidence. A vocabulary-set check + is also insufficient: ``Alice approved alpha, not beta`` reuses every token in + ``Alice approved beta, not alpha`` while reversing its meaning. A general + entailment checker would require another fallible model, so the safe offline + verifier accepts only an exact ordered source span after removing a narrow set + of citation glue words. Legitimate paraphrases that fail this conservative check + degrade to the deterministic extractive answer instead of being labelled grounded. + """ + if not _citations_are_valid(text, len(citations)): + return False + sources = { + int(citation["n"]): _ordered_tokens(str(citation.get("content", ""))) + for citation in citations + if isinstance(citation.get("n"), int) + } + clauses = [clause.strip() for clause in re.split(r"(?<=[.!?])\s+", text) if clause.strip()] + if not clauses: + return False + for clause in clauses: + markers = [int(marker) for marker in _CITE_RE.findall(clause)] + if not markers: + return False + claim_tokens = [ + token for token in _ordered_tokens(_CITE_RE.sub("", clause)) + if token not in _SYNTHESIS_GLUE_TERMS + ] + if not claim_tokens or not any( + _contains_span(sources.get(marker, []), claim_tokens) + for marker in markers + ): + return False + return True def build_grounded_answer(query: str, result: RecallResult, embedder, *, @@ -127,12 +247,49 @@ def build_grounded_answer(query: str, result: RecallResult, embedder, *, Deterministic and offline unless an ``LLM`` is injected. Never raises on LLM failure — it degrades to the extractive answer. """ - # Score support over ALL retrieved memories (not just the first max_citations) so a - # strongly-supporting memory ranked lower by the fused recall score still counts. - chunks = list(result.chunks) + try: + min_support = float(min_support) + except (TypeError, ValueError) as exc: + raise ValueError("min_support must be a finite number between 0 and 1") from exc + if not math.isfinite(min_support) or not 0.0 <= min_support <= 1.0: + raise ValueError("min_support must be a finite number between 0 and 1") + try: + max_citations = int(max_citations) + except (TypeError, ValueError) as exc: + raise ValueError("max_citations must be a positive integer") from exc + if max_citations < 1: + raise ValueError("max_citations must be a positive integer") + + # Grounding may use only evidence the ContextPacker actually admitted. Raw retrieval + # candidates can be omitted or truncated by the caller's token budget and therefore + # are not evidence available to the answerer. + raw_by_id = {str(chunk.get("id")): chunk for chunk in result.chunks} + chunks = [] + for packed in result.packed_chunks: + raw = raw_by_id.get(str(packed.id)) + if raw is None or not packed.excerpt: + continue + chunks.append({**raw, "content": packed.excerpt}) contents = [str(c.get("content", "")) for c in chunks] per = _support_scores(query, contents, embedder) support = max(per) if per else 0.0 + count_answer_tokens = result.token_counter or RegexTokenCounter() + budget_tokens = result.usage.budget_tokens if result.usage is not None else 0 + recall_metadata = { + "usage": asdict(result.usage) if result.usage is not None else {}, + "packed_sources": [{ + "id": packed.id, + "tokens": packed.tokens, + "truncated": packed.truncated, + "reason": packed.reason, + } for packed in result.packed_chunks], + "valid_at": result.valid_at, + "known_at": result.known_at, + "historical": result.historical, + "retrieval_profile": result.retrieval_profile, + "retrieval_trace": result.retrieval_trace, + } + recall_metadata["usage"]["answer_tokens"] = 0 if not chunks or support < min_support: return GroundedAnswer( @@ -140,6 +297,7 @@ def build_grounded_answer(query: str, result: RecallResult, embedder, *, reason=(f"no memory in scope sufficiently supports this query " f"(support {support:.3f} < floor {min_support:.3f}); " f"not answering rather than guessing"), + **recall_metadata, ) # Cite the sources that individually clear the floor, strongest evidence first, capped @@ -159,20 +317,36 @@ def build_grounded_answer(query: str, result: RecallResult, embedder, *, stripped = (prose or "").strip() if stripped == ABSTAIN_SENTINEL: return GroundedAnswer(grounded=False, abstained=True, support=support, - reason="synthesiser judged the sources insufficient") - # Accept synthesised prose only if it actually cites a source; otherwise it may - # have introduced an uncited (possibly fabricated) claim, so fall back to the - # deterministic extractive answer — grounded and cited by construction. - if stripped and _cites_a_source(stripped, len(citations)): + reason="synthesiser judged the sources insufficient", + **recall_metadata) + # Markers alone are not evidence: an LLM can write "Invented fact [1]". + # Accept prose only after the deterministic, citation-specific source + # vocabulary check; otherwise return extractive evidence by construction. + answer_tokens = count_answer_tokens(stripped) + if ( + stripped + and answer_tokens <= budget_tokens + and _synthesis_is_source_bounded(stripped, citations) + ): + recall_metadata["usage"]["answer_tokens"] = answer_tokens return GroundedAnswer(answer=stripped, grounded=True, abstained=False, support=support, synthesized=True, - citations=citations) + citations=citations, **recall_metadata) except Exception: pass # any LLM failure -> fall through to the deterministic answer - return GroundedAnswer(answer=_extractive_answer(citations), grounded=True, + extractive = _extractive_answer(citations) + answer_tokens = count_answer_tokens(extractive) + if answer_tokens > budget_tokens: + return GroundedAnswer( + grounded=False, abstained=True, support=support, + reason="packed evidence cannot fit a cited answer within the token budget", + **recall_metadata, + ) + recall_metadata["usage"]["answer_tokens"] = answer_tokens + return GroundedAnswer(answer=extractive, grounded=True, abstained=False, support=support, synthesized=False, - citations=citations) + citations=citations, **recall_metadata) def _extractive_answer(citations: list[dict]) -> str: @@ -182,8 +356,10 @@ def _extractive_answer(citations: list[dict]) -> str: for c in citations: text = " ".join(str(c.get("content", "")).split()) title = str(c.get("title", "")).strip() - prefix = f"{title}: " if title else "" - lines.append(f"[{c['n']}] {prefix}{text}") + header = f"[{c['n']}]" + if title: + header += " " + " ".join(title.split())[:120] + lines.append(f"{header}\n{text}") return "\n".join(lines) diff --git a/engraphis/core/interfaces.py b/engraphis/core/interfaces.py index b0596d12..847161e6 100644 --- a/engraphis/core/interfaces.py +++ b/engraphis/core/interfaces.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import math from dataclasses import dataclass, field from enum import Enum from typing import Any, Iterable, Literal, Optional, Protocol, runtime_checkable @@ -40,6 +41,21 @@ class GraphLayer(str, Enum): SEMANTIC = "semantic" +def _finite_timestamp(value: Optional[float], name: str) -> Optional[float]: + """Normalize public temporal anchors and reject SQLite's non-finite values.""" + if value is None: + return None + if isinstance(value, bool): + raise ValueError(f"{name} must be a finite timestamp") + try: + timestamp = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a finite timestamp") from exc + if not math.isfinite(timestamp): + raise ValueError(f"{name} must be a finite timestamp") + return timestamp + + # ── Records ────────────────────────────────────────────────────────────────── @dataclass @@ -65,10 +81,13 @@ class MemoryRecord: valid_to: Optional[float] = None # world-time: when it stopped being true ingested_at: Optional[float] = None # system-time: when we learned it expired_at: Optional[float] = None # system-time: when we retired it + subject_key: str = "" # stable optional claim subject + claim_kind: str = "" # optional claim predicate/category pinned: bool = False sensitivity: str = "normal" # normal | sensitive | secret provenance: dict[str, Any] = field(default_factory=dict) embedding: Optional[np.ndarray] = None + valid_to_recorded_at: Optional[float] = None # when valid_to was learned @dataclass @@ -80,11 +99,34 @@ class SearchFilter: scopes: Optional[list[Scope]] = None mtypes: Optional[list[MemoryType]] = None graph_layers: Optional[list[GraphLayer]] = None - as_of: Optional[float] = None # bi-temporal time anchor; None = now + # ``as_of`` remains a compatibility alias for the world-time ``valid_at`` + # anchor. New callers can independently select what was true and what + # had been learned at that time. + as_of: Optional[float] = None # Contextual recall sees broader scopes as ancestors: a repo read can see that # repo plus workspace/user memories, and a session read can additionally see its # exact session. Storage/governance queries stay exact unless they opt in. include_ancestors: bool = False + # Appended after every 1.x field so positional construction remains compatible. + valid_at: Optional[float] = None + known_at: Optional[float] = None + + def __post_init__(self) -> None: + self.as_of = _finite_timestamp(self.as_of, "as_of") + self.valid_at = _finite_timestamp(self.valid_at, "valid_at") + self.known_at = _finite_timestamp(self.known_at, "known_at") + if self.as_of is not None and self.valid_at is not None: + if self.as_of != self.valid_at: + raise ValueError("as_of and valid_at must match when both are supplied") + # Keep legacy backends that read ``as_of`` correct as callers move to + # the less ambiguous ``valid_at`` name. + self.valid_at = self.valid_at if self.valid_at is not None else self.as_of + self.as_of = self.valid_at + + @property + def historical(self) -> bool: + """Whether either time axis was explicitly anchored by the caller.""" + return self.valid_at is not None or self.known_at is not None @dataclass @@ -96,6 +138,29 @@ class Candidate: record: Optional[MemoryRecord] = None +@dataclass +class PackedChunk: + """One source excerpt selected by a context-packing implementation.""" + id: str + excerpt: str + tokens: int + truncated: bool = False + reason: str = "" + + +@dataclass +class ContextUsage: + """Token accounting emitted by a context-packing implementation.""" + budget_tokens: int + context_tokens: int + source_tokens: int + saved_tokens: int + savings_ratio: float + packed_count: int + omitted_count: int + token_counter: str = "estimate_tokens" + + @dataclass class Node: """A knowledge-graph node (entity or concept).""" @@ -123,6 +188,7 @@ class Edge: ingested_at: Optional[float] = None expired_at: Optional[float] = None provenance: dict[str, Any] = field(default_factory=dict) + valid_to_recorded_at: Optional[float] = None @dataclass @@ -219,6 +285,20 @@ class Reranker(Protocol): def rerank(self, query: str, candidates: list[Candidate], k: int) -> list[Candidate]: ... +@runtime_checkable +class ContextPacker(Protocol): + """Choose budgeted, explainable source excerpts for an agent context.""" + def pack(self, query: str, candidates: list[Candidate], token_budget: int + ) -> tuple[str, list[PackedChunk], ContextUsage]: ... + def count_tokens(self, text: str) -> int: ... + + +@runtime_checkable +class RetrievalPolicy(Protocol): + """Select a named retrieval profile without coupling core to a backend.""" + def profile(self, query: str) -> str: ... + + @runtime_checkable class LLM(Protocol): """External or local model for synthesis and structured extraction (§8.2).""" diff --git a/engraphis/core/recall.py b/engraphis/core/recall.py index 468c57a6..4b7a1300 100644 --- a/engraphis/core/recall.py +++ b/engraphis/core/recall.py @@ -12,18 +12,30 @@ """ from __future__ import annotations +import inspect import re -from dataclasses import dataclass, field -from typing import Optional +from dataclasses import dataclass, field, replace +from typing import Any, Callable, Optional from engraphis.core import scoring +from engraphis.core.context import DeterministicContextPacker from engraphis.core.graphrank import personalized_pagerank from engraphis.core.interfaces import ( Candidate, + ContextPacker, + ContextUsage, MemoryRecord, + PackedChunk, Reranker, + RetrievalPolicy, SearchFilter, ) +from engraphis.core.retrieval_policy import ( + DeterministicRetrievalPolicy, + ProfileConfig, + RETRIEVAL_PROFILES, + profile_config, +) from engraphis.core.store import Store, memory_matches_filter, now_ts @@ -32,12 +44,22 @@ class RecallResult: chunks: list[dict] = field(default_factory=list) context: str = "" count: int = 0 + packed_chunks: list[PackedChunk] = field(default_factory=list) + usage: Optional[ContextUsage] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None + historical: bool = False + retrieval_profile: str = "balanced" + retrieval_trace: Optional[list[dict[str, Any]]] = None + token_counter: Optional[Callable[[str], int]] = field(default=None, repr=False) class RecallEngine: def __init__(self, store: Store, embedder, vector_index, reranker: Optional[Reranker] = None, *, weights: Optional[dict] = None, recency_tau_days: float = 30.0, - token_budget: int = 1500, graph_mode: str = "ppr") -> None: + token_budget: int = 1500, graph_mode: str = "ppr", + context_packer: Optional[ContextPacker] = None, + retrieval_policy: Optional[RetrievalPolicy] = None) -> None: self.store = store self.embedder = embedder self.index = vector_index @@ -45,27 +67,73 @@ def __init__(self, store: Store, embedder, vector_index, reranker: Optional[Rera self.weights = weights or scoring.DEFAULT_WEIGHTS self.recency_tau_days = recency_tau_days self.token_budget = token_budget + self.context_packer = context_packer or DeterministicContextPacker() + self.retrieval_policy = retrieval_policy or DeterministicRetrievalPolicy() # "ppr" (default) = Personalized PageRank over entities+links (multi-hop); # "1hop" = the Phase-1 entity expansion, kept for fallback and ablation. self.graph_mode = graph_mode def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, - candidate_k: int = 50, reinforce: bool = True) -> RecallResult: + candidate_k: int = 50, reinforce: bool = False, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", + diagnostics: bool = False, + arm_config: Optional[ProfileConfig] = None) -> RecallResult: flt = flt or SearchFilter() - now = flt.as_of if flt.as_of is not None else now_ts() + requested_historical = flt.historical + snapshot = now_ts() + effective_valid_at = ( + flt.valid_at if flt.valid_at is not None else snapshot + ) + effective_known_at = ( + flt.known_at if flt.known_at is not None else snapshot + ) + flt = replace( + flt, + as_of=effective_valid_at, + valid_at=effective_valid_at, + known_at=effective_known_at, + ) + now = effective_valid_at + budget = self.token_budget if token_budget is None else max(0, int(token_budget)) + requested_profile = str(retrieval_profile or "balanced").strip().casefold() + if requested_profile not in RETRIEVAL_PROFILES: + choices = ", ".join(sorted(RETRIEVAL_PROFILES)) + raise ValueError(f"retrieval_profile must be one of: {choices}") + selected_profile = ( + self.retrieval_policy.profile(query) + if requested_profile == "auto" + else requested_profile + ) + # ``arm_config`` is a composition-time override for controlled offline + # ablations. Normal callers still use only named RetrievalPolicy profiles, + # so benchmark labels do not expand the public routing contract. + config = arm_config or profile_config(selected_profile) # ── arms ───────────────────────────────────────────────────────────── - qvec = self.embedder.embed([query])[0] - vec = dict(self.index.search(qvec, candidate_k, filter=flt)) # id -> cosine - lex = dict(self.store.fts_search(query, candidate_k, filter=flt)) # id -> lexical - graph = self._graph_arm(query, flt, now) # id -> weight + if config.vector: + qvec = self.embedder.embed([query])[0] + vec = dict(self.index.search(qvec, candidate_k, filter=flt)) + else: + vec = {} + lex = ( + dict(self.store.fts_search(query, candidate_k, filter=flt)) + if config.lexical else {} + ) + graph = self._graph_arm(query, flt, now, candidate_k=candidate_k) if config.graph else {} + code = ( + self._code_arm( + query, flt, candidate_k, historical=requested_historical + ) + if config.code else {} + ) # ── gather candidates and enforce visibility defensively ───────────── # Sorted, not raw set order: a set of ids iterates in hash order, which varies with # PYTHONHASHSEED, so equal-scored results used to come back in a different order in # every process. Sorting here (and on the final sort below) makes recall reproducible. # One batched lookup replaces ~150 single-row get_memory() calls per recall. - candidate_ids = sorted(set(vec) | set(lex) | set(graph)) + candidate_ids = sorted(set(vec) | set(lex) | set(graph) | set(code)) fetched = self.store.get_memories(candidate_ids) recs: dict[str, MemoryRecord] = {} for mid in candidate_ids: @@ -73,35 +141,132 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, if rec and memory_matches_filter(rec, flt, at=now): recs[mid] = rec if not recs: - return RecallResult() + context, packed, usage = self.context_packer.pack(query, [], budget) + return RecallResult( + context=context, + packed_chunks=packed, + usage=usage, + valid_at=flt.valid_at, + known_at=flt.known_at, + historical=requested_historical, + retrieval_profile=selected_profile, + retrieval_trace=[] if diagnostics else None, + token_counter=getattr(self.context_packer, "count_tokens", None), + ) sem_n = scoring.normalize({i: vec[i] for i in vec if i in recs}) lex_n = scoring.normalize({i: lex[i] for i in lex if i in recs}) grp_n = scoring.normalize({i: graph[i] for i in graph if i in recs}) + code_n = scoring.normalize({i: code[i] for i in code if i in recs}) rrf = scoring.reciprocal_rank_fusion([ - _ranked(vec, recs), _ranked(lex, recs), _ranked(graph, recs), + ranked for ranked in ( + _ranked(vec, recs), + _ranked(lex, recs), + _ranked(graph, recs), + _ranked(code, recs), + ) if ranked ]) # ── six-term weighted score (+ small RRF nudge for cross-arm agreement) ── scored: list[Candidate] = [] + score_details: dict[str, dict[str, Any]] = {} for mid, rec in recs.items(): w = self.weights.get(rec.mtype, scoring.Weights()) + adjusted_semantic = sem_n.get(mid, 0.0) * config.semantic_scale + adjusted_lexical = lex_n.get(mid, 0.0) * config.lexical_scale + adjusted_graph = ( + grp_n.get(mid, 0.0) * config.graph_scale + + (config.graph_presence_bonus if mid in graph else 0.0) + ) + adjusted_code = ( + code_n.get(mid, 0.0) * config.code_scale + + (config.code_presence_bonus if mid in code else 0.0) + ) + semantic_score = max(adjusted_semantic, adjusted_code) base = scoring.score_memory( rec, now=now, weights=w, - semantic=sem_n.get(mid, 0.0), lexical=lex_n.get(mid, 0.0), - graph=grp_n.get(mid, 0.0), recency_tau_days=self.recency_tau_days, + semantic=semantic_score, lexical=adjusted_lexical, + graph=adjusted_graph, recency_tau_days=self.recency_tau_days, + ) + arms = [ + name for name, values in ( + ("semantic", vec), + ("lexical", lex), + ("graph", graph), + ("code", code), + ) if mid in values + ] + fusion_score = base + 0.5 * rrf.get(mid, 0.0) + arm = ( + "code" if "code" in arms + else (arms[0] if len(arms) == 1 else ("hybrid" if arms else "fused")) ) - arm = "semantic" if mid in vec else ("lexical" if mid in lex else "graph") - scored.append(Candidate(id=mid, score=base + 0.5 * rrf.get(mid, 0.0), - arm=arm, record=rec)) + scored.append(Candidate( + id=mid, score=fusion_score, arm=arm, record=rec + )) + score_details[mid] = { + "raw": { + "semantic": vec.get(mid), + "lexical": lex.get(mid), + "graph": graph.get(mid), + "code": code.get(mid), + }, + "normalized": { + "semantic": sem_n.get(mid, 0.0), + "lexical": lex_n.get(mid, 0.0), + "graph": grp_n.get(mid, 0.0), + "code": code_n.get(mid, 0.0), + }, + "profile_adjusted": { + "semantic": adjusted_semantic, + "lexical": adjusted_lexical, + "graph": adjusted_graph, + "code": adjusted_code, + }, + "six_term_score": base, + "rrf_score": rrf.get(mid, 0.0), + "fusion_score": fusion_score, + "rerank_score": None, + "calibrated_score": fusion_score, + "arm_agreement": len(arms), + "arms": arms, + } # Tie-break on id so equal scores get a stable, process-independent order. scored.sort(key=lambda c: (-c.score, c.id)) # ── rerank top-N, keep k ───────────────────────────────────────────── pool = scored[: max(k * 4, k)] - final = self.reranker.rerank(query, pool, k) if self.reranker else pool[:k] - - if reinforce: + if self.reranker: + fused_before = {candidate.id: candidate.score for candidate in pool} + reranked = self.reranker.rerank(query, pool, k) + rerank_raw = { + candidate.id: float(candidate.score) for candidate in reranked + } + changed = any( + abs(rerank_raw[candidate.id] - fused_before.get(candidate.id, 0.0)) > 1e-12 + for candidate in reranked + ) + if changed: + fusion_norm = scoring.normalize({ + candidate.id: fused_before.get(candidate.id, 0.0) + for candidate in reranked + }) + rerank_norm = scoring.normalize(rerank_raw) + for candidate in reranked: + candidate.score = ( + 0.7 * fusion_norm.get(candidate.id, 0.0) + + 0.3 * rerank_norm.get(candidate.id, 0.0) + ) + reranked.sort(key=lambda candidate: (-candidate.score, candidate.id)) + final = reranked[:k] + for candidate in final: + detail = score_details[candidate.id] + detail["rerank_score"] = rerank_raw.get(candidate.id) + detail["calibrated_score"] = candidate.score + else: + final = pool[:k] + + if reinforce and not requested_historical: for c in final: self.store.reinforce(c.id, boost=scoring.INTERACTION_BOOST["recall"]) @@ -109,25 +274,204 @@ def recall(self, query: str, flt: Optional[SearchFilter] = None, *, k: int = 8, "id": c.id, "title": c.record.title, "content": c.record.content, "scope": c.record.scope.value, "mtype": c.record.mtype.value, "repo_id": c.record.repo_id, "score": round(c.score, 4), "arm": c.arm, + "subject_key": c.record.subject_key, + "claim_kind": c.record.claim_kind, "retention": round(scoring.retention(c.record.stability, c.record.last_access, now), 4), "provenance": c.record.provenance, } for c in final] - return RecallResult(chunks=chunks, context=self._pack(final), count=len(final)) + context, packed_chunks, usage = self.context_packer.pack(query, final, budget) + trace = None + if diagnostics: + trace = [ + {"id": candidate.id, **score_details[candidate.id]} + for candidate in final + ] + return RecallResult( + chunks=chunks, + context=context, + count=len(final), + packed_chunks=packed_chunks, + usage=usage, + valid_at=flt.valid_at, + known_at=flt.known_at, + historical=requested_historical, + retrieval_profile=selected_profile, + retrieval_trace=trace, + token_counter=getattr(self.context_packer, "count_tokens", None), + ) # ── arms / helpers ──────────────────────────────────────────────────────── - def _graph_arm(self, query: str, flt: SearchFilter, now: float) -> dict[str, float]: - if self.graph_mode == "1hop": - return self._graph_arm_1hop(query, flt, now) - return self._graph_arm_ppr(query, flt, now) + def _code_arm( + self, + query: str, + flt: SearchFilter, + candidate_k: int, + *, + historical: Optional[bool] = None, + ) -> dict[str, float]: + """Bridge code-symbol matches to scoped memories with bounded work. + + The symbol graph remains optional: an unindexed repo simply contributes + no candidates. Query fan-out, matched symbols, graph edges, and linked + memories are all capped so code recall cannot degrade into a repository + scan. + """ + if not flt.repo_id: + return {} + identifiers = [] + seen_identifiers = set() + stop = { + "about", "called", "class", "code", "does", "file", "from", + "function", "into", "module", "that", "this", "what", "where", + "which", "with", + } + for value in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", query): + folded = value.casefold() + if folded in stop or folded in seen_identifiers: + continue + seen_identifiers.add(folded) + identifiers.append(value) + if len(identifiers) >= 8: + break + if not identifiers: + return {} - def _graph_arm_ppr(self, query: str, flt: SearchFilter, now: float) -> dict[str, float]: + symbols: dict[str, dict] = {} + symbol_strength: dict[str, float] = {} + per_term = max(2, min(12, candidate_k // max(1, len(identifiers)))) + for identifier in identifiers: + matches = _call_temporal_store( + self.store.search_symbols, + flt, + flt.repo_id, + identifier, + limit=per_term, + requested_historical=historical, + ) + for rank, symbol in enumerate(matches): + symbol_id = symbol.get("id") + if not symbol_id: + continue + exact = identifier.casefold() in { + str(symbol.get("name") or "").casefold(), + str(symbol.get("fqname") or "").casefold(), + } + strength = (1.0 if exact else 0.75) / (rank + 1) + symbols[symbol_id] = symbol + symbol_strength[symbol_id] = max( + symbol_strength.get(symbol_id, 0.0), strength + ) + if not symbols: + return {} + + # Expand one stored code edge to capture callers/callees, bounded by a + # multiple of candidate_k. Both symbol ids and parser-emitted names are + # accepted because language backends use both representations. + code_edges = _call_temporal_store( + self.store.list_code_edges, + flt, + flt.repo_id, + limit=max(100, min(2000, candidate_k * 20)), + layers=flt.graph_layers, + requested_historical=historical, + ) + aliases: dict[str, str] = {} + for symbol_id, symbol in symbols.items(): + for key in ("id", "name", "fqname"): + value = str(symbol.get(key) or "") + if value: + aliases[value] = symbol_id + related_names: dict[str, float] = {} + for edge in code_edges: + src, dst = str(edge.get("src") or ""), str(edge.get("dst") or "") + if src in aliases: + related_names[dst] = max( + related_names.get(dst, 0.0), + symbol_strength[aliases[src]] * 0.55, + ) + if dst in aliases: + related_names[src] = max( + related_names.get(src, 0.0), + symbol_strength[aliases[dst]] * 0.55, + ) + if related_names: + all_symbols = _call_temporal_store( + self.store.list_symbols, + flt, + flt.repo_id, + limit=max(100, min(2000, candidate_k * 20)), + requested_historical=historical, + ) + for symbol in all_symbols: + matched_strength = max( + ( + related_names.get(str(symbol.get(key) or ""), 0.0) + for key in ("id", "name", "fqname") + ), + default=0.0, + ) + symbol_id = symbol.get("id") + if matched_strength > 0.0 and symbol_id: + symbols[symbol_id] = symbol + symbol_strength[symbol_id] = max( + symbol_strength.get(symbol_id, 0.0), matched_strength + ) + + selected_symbol_ids = sorted( + symbols, + key=lambda value: (-symbol_strength.get(value, 0.0), value), + )[:max(10, min(100, candidate_k * 2))] + rows_by_symbol = _call_temporal_store( + self.store.memories_for_symbols, + flt, + flt.repo_id, + selected_symbol_ids, + limit=max(2, min(10, candidate_k)), + requested_historical=historical, + ) + out: dict[str, float] = {} + for symbol_id in selected_symbol_ids: + rows = rows_by_symbol.get(symbol_id, []) + for rank, row in enumerate(rows): + memory_id = row.get("id") + if not memory_id: + continue + confidence = max(0.0, min(1.0, float(row.get("confidence") or 0.0))) + score = symbol_strength[symbol_id] * confidence / (rank + 1) + out[memory_id] = max(out.get(memory_id, 0.0), score) + return dict( + sorted(out.items(), key=lambda item: (-item[1], item[0]))[:candidate_k] + ) + + def _graph_arm( + self, + query: str, + flt: SearchFilter, + now: float, + *, + candidate_k: int = 50, + ) -> dict[str, float]: + if flt.graph_layers is not None and not flt.graph_layers: + return {} + if self.graph_mode == "1hop": + return self._graph_arm_1hop(query, flt, now, candidate_k=candidate_k) + return self._graph_arm_ppr(query, flt, now, candidate_k=candidate_k) + + def _graph_arm_ppr( + self, + query: str, + flt: SearchFilter, + now: float, + *, + candidate_k: int = 50, + ) -> dict[str, float]: """Personalized PageRank arm: build the scoped entity/memory graph — entity↔entity edges (bi-temporal), memory↔entity mentions, memory↔memory links — seed at the query's entities, and rank memories by walk probability. Multi-hop associations surface without expanding an explicit hop count; entity nodes are prefixed so names can never collide with memory ids.""" - entity_map = self._entity_map(flt) + entity_map = self._seed_entity_map(query, flt) patterns = { eid: (name.casefold(), _entity_pattern(name)) for eid, name in entity_map.items() @@ -149,37 +493,53 @@ def connect(a: str, b: str, w: float) -> None: adj.setdefault(a, []).append((b, w)) adj.setdefault(b, []).append((a, w)) - for e in self.store.edges_in_scope(flt, at=now): + edges = self.store.edges_in_scope(flt, at=now, limit=4000) + for e in edges: connect(ent(e.src), ent(e.dst), max(float(e.weight or 1.0), 1e-6)) - # Past this cap, PPR would be rejected below anyway. Fall back before - # scanning every memory against every entity and then repeating that - # work in the 1-hop arm. - if len(adj) > 4000: - return self._graph_arm_1hop(query, flt, now) - - recs = self.store.list_memories(flt, limit=500) - for rec in recs: - hay = f"{rec.title} {rec.content}" - hay_folded = hay.casefold() - for eid, (needle, pattern) in patterns.items(): - # Most entity names are absent. The C-level substring guard avoids - # millions of comparatively expensive regex searches while the - # regex retains exact token-boundary semantics for actual matches. - if needle in hay_folded and pattern.search(hay): - connect(rec.id, ent(eid), 1.0) + incidence = self.store.list_memory_entities(flt, limit=12_000) + memory_ids = sorted({ + str(row.get("memory_id") or "") + for row in incidence if row.get("memory_id") + }) + incidence_strength: dict[tuple[str, str], float] = {} + for row in incidence: + memory_id = str(row.get("memory_id") or "") + entity_id = str(row.get("entity_id") or "") + if memory_id and entity_id: + key = (memory_id, entity_id) + incidence_strength[key] = max( + incidence_strength.get(key, 0.0), + max(float(row.get("confidence") or 0.0), 1e-6), + ) + for (memory_id, entity_id), confidence in incidence_strength.items(): + connect(memory_id, ent(entity_id), confidence) for link in self.store.links_among( - [r.id for r in recs], layers=flt.graph_layers + memory_ids, + layers=flt.graph_layers, + flt=flt, + limit=20_000, ): connect(link["a"], link["b"], 1.0) ranked = personalized_pagerank(adj, [ent(eid) for eid in seeds]) - return {nid: score for nid, score in ranked.items() - if not nid.startswith("ent::") and score > 0.0} - - def _graph_arm_1hop(self, query: str, flt: SearchFilter, now: float) -> dict[str, float]: - entity_map = self._entity_map(flt) + memory_scores = [ + (nid, score) for nid, score in ranked.items() + if not nid.startswith("ent::") and score > 0.0 + ] + memory_scores.sort(key=lambda item: (-item[1], item[0])) + return dict(memory_scores[:max(0, int(candidate_k))]) + + def _graph_arm_1hop( + self, + query: str, + flt: SearchFilter, + now: float, + *, + candidate_k: int = 50, + ) -> dict[str, float]: + entity_map = self._seed_entity_map(query, flt) patterns = { eid: (name.casefold(), _entity_pattern(name)) for eid, name in entity_map.items() @@ -193,31 +553,42 @@ def _graph_arm_1hop(self, query: str, flt: SearchFilter, now: float) -> dict[str ] if not seed_ids: return {} - names = {entity_map[eid] for eid in seed_ids if entity_map.get(eid)} - for e in self.store.neighbors(seed_ids, at=now, layers=flt.graph_layers): - if e.src in entity_map: - names.add(entity_map[e.src]) - if e.dst in entity_map: - names.add(entity_map[e.dst]) + related_ids = set(seed_ids) + for edge in self.store.neighbors( + seed_ids, at=now, layers=flt.graph_layers, flt=flt + ): + related_ids.add(edge.src) + related_ids.add(edge.dst) + rows = self.store.list_memory_entities( + flt, entity_ids=sorted(related_ids), limit=12_000 + ) out: dict[str, float] = {} - name_patterns = [ - (name.casefold(), _entity_pattern(name)) - for name in names - if name - ] - for rec in self.store.list_memories(flt, limit=500): - hay = f"{rec.title} {rec.content}" - hay_folded = hay.casefold() - hits = sum( - 1 - for needle, pattern in name_patterns - if needle in hay_folded and pattern.search(hay) - ) - if hits: - out[rec.id] = float(hits) - return out - - def _entity_map(self, flt: SearchFilter) -> dict[str, str]: + if rows: + for row in rows: + memory_id = str(row.get("memory_id") or "") + if memory_id: + out[memory_id] = ( + out.get(memory_id, 0.0) + + max(0.0, float(row.get("confidence") or 0.0)) + ) + return dict(sorted( + out.items(), key=lambda item: (-item[1], item[0]) + )[:max(0, int(candidate_k))]) + + return dict(sorted( + out.items(), key=lambda item: (-item[1], item[0]) + )[:max(0, int(candidate_k))]) + + def _seed_entity_map( + self, query: str, flt: SearchFilter, *, limit: int = 2048, + ) -> dict[str, str]: + """Return a bounded, scoped set of entity names that may occur in ``query``.""" + terms = sorted({ + term.casefold() for term in re.findall(r"[\w@#.+-]+", query) + if len(term) >= 2 + })[:16] + if not terms: + return {} sql = "SELECT DISTINCT id, name FROM entities" clauses, params = [], [] if flt.workspace_id: @@ -235,23 +606,53 @@ def _entity_map(self, flt: SearchFilter) -> dict[str, str]: else: clauses.append("repo_id=?") params.append(flt.repo_id) + clauses.append( + "(" + " OR ".join("instr(lower(name), ?) > 0" for _ in terms) + ")" + ) + params.extend(terms) if clauses: sql += " WHERE " + " AND ".join(clauses) - return {r["id"]: r["name"] for r in self.store.conn.execute(sql, params).fetchall()} + sql += " ORDER BY id LIMIT ?" + params.append(max(0, int(limit))) + return { + r["id"]: r["name"] + for r in self.store.conn.execute(sql, params).fetchall() + } + + def _entity_map(self, flt: SearchFilter, *, limit: int = 2048) -> dict[str, str]: + """Compatibility view of scoped entities without restoring unbounded recall scans. + + The retrieval pipeline uses :meth:`_seed_entity_map` so graph seeding remains + query-directed. Older integrations and scope-invariant tests exercised this private + helper directly, so retain its original semantics behind an explicit safety bound. + """ + sql = "SELECT DISTINCT id, name FROM entities" + clauses, params = [], [] + if flt.workspace_id: + if flt.include_ancestors: + clauses.append("(workspace_id=? OR workspace_id IS NULL)") + else: + clauses.append("workspace_id=?") + params.append(flt.workspace_id) + if flt.repo_id: + if flt.include_ancestors: + clauses.append("(repo_id=? OR repo_id IS NULL)") + else: + clauses.append("repo_id=?") + params.append(flt.repo_id) + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY id LIMIT ?" + params.append(max(0, int(limit))) + return { + row["id"]: row["name"] + for row in self.store.conn.execute(sql, params).fetchall() + } def _pack(self, cands: list[Candidate]) -> str: - parts, used = [], 0 - for c in cands: - r = c.record - header = f"[{r.scope.value}:{r.repo_id or '-'}]" - if r.title: - header += f" {r.title}" - block = f"{header}\n{r.summary or r.content}" - used += len(block) // 4 - if used > self.token_budget and parts: - break - parts.append(block) - return "\n\n".join(parts) + """Compatibility helper for callers that exercised the old private method.""" + context, _, _ = self.context_packer.pack("", cands, self.token_budget) + return context def _entity_pattern(name: str) -> re.Pattern[str]: @@ -263,3 +664,34 @@ def _ranked(arm: dict[str, float], recs: dict) -> list[str]: # Tie-break on id: RRF depends on rank position, so equal arm scores must not order # differently between runs (they feed the final score). return [i for i, _ in sorted(arm.items(), key=lambda x: (-x[1], x[0])) if i in recs] + + +def _call_temporal_store( + method, + flt: SearchFilter, + *args, + requested_historical: Optional[bool] = None, + **kwargs, +): + """Call an optional code-store extension without masking implementation bugs. + + Older third-party stores may not expose the v5 ``flt`` keyword. Current reads can + retain their legacy behavior, but historical reads must fail closed: retrying a + method without the filter would silently substitute present-day code evidence. + Signature inspection distinguishes an unsupported keyword from a genuine + ``TypeError`` raised inside the implementation, which is allowed to propagate. + """ + try: + parameters = inspect.signature(method).parameters.values() + supports_filter = any( + parameter.name == "flt" + or parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + supports_filter = False + if supports_filter: + return method(*args, flt=flt, **kwargs) + if flt.historical if requested_historical is None else requested_historical: + return [] + return method(*args, **kwargs) diff --git a/engraphis/core/resolve.py b/engraphis/core/resolve.py index 5d8afe96..364df3d7 100644 --- a/engraphis/core/resolve.py +++ b/engraphis/core/resolve.py @@ -39,12 +39,18 @@ # the new fact and reinforce the stale one, while a wrongly-INVALIDATE'd restatement just # refreshes the phrasing and keeps the old version readable in history. PARAPHRASE_EMBED_SIM = 0.90 +# Supersession without an explicit claim key is intentionally stricter than a +# generic "related" judgment. Both independent signals must agree before a +# write hides a currently-live fact from ordinary recall. +STRONG_SUBJECT_TOKEN_JACCARD = 0.55 +STRONG_JOINT_EMBED_SIM = 0.45 class ResolutionOp(str, Enum): ADD = "add" # genuinely new -> insert NOOP = "noop" # already known -> reinforce the existing memory, don't insert INVALIDATE = "invalidate" # same subject, new content -> close old, insert new + RELATE = "relate" # retain both facts and persist a semantic relation @dataclass(frozen=True) @@ -54,7 +60,8 @@ class Resolution: reason: str = "" -def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]]) -> Resolution: +def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]], *, + subject_key: str = "", claim_kind: str = "") -> Resolution: """Decide ADD / NOOP / INVALIDATE for new content against its nearest neighbors. ``neighbors`` are ``(embedding_similarity, MemoryRecord)`` pairs that the caller has @@ -65,11 +72,29 @@ def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]]) -> the embedding cosine as a second signal for paraphrased restatements/contradictions. """ cand_tokens = tokenize(candidate_text) - best: Optional[tuple[float, MemoryRecord, float]] = None # (overlap, rec, sim) - best_sim: Optional[tuple[float, MemoryRecord]] = None # highest-cosine neighbor + candidate_subject = str(subject_key or "").strip() + candidate_kind = str(claim_kind or "").strip() + exact_claim_neighbors: list[tuple[float, MemoryRecord]] = [] + fallback_neighbors: list[tuple[float, MemoryRecord]] = [] for sim, rec in neighbors: + record_subject = str(rec.subject_key or "").strip() + record_kind = str(rec.claim_kind or "").strip() + # Explicit claim identities outrank similarity. Two keyed records that + # disagree on subject or predicate cannot be duplicate/supersession + # candidates merely because their prose happens to be similar. + if candidate_subject and record_subject: + if candidate_subject != record_subject or candidate_kind != record_kind: + continue + exact_claim_neighbors.append((sim, rec)) + continue if sim < RELATED_SIM_FLOOR: continue + fallback_neighbors.append((sim, rec)) + + considered = exact_claim_neighbors or fallback_neighbors + best: Optional[tuple[float, MemoryRecord, float]] = None # (overlap, rec, sim) + best_sim: Optional[tuple[float, MemoryRecord]] = None # highest-cosine neighbor + for sim, rec in considered: overlap = jaccard(cand_tokens, tokenize(f"{rec.title} {rec.content}")) if best is None or overlap > best[0]: best = (overlap, rec, sim) @@ -80,20 +105,47 @@ def resolve(candidate_text: str, neighbors: list[tuple[float, MemoryRecord]]) -> return Resolution(ResolutionOp.ADD, reason="no related memory in scope") overlap, rec, sim = best + same_subject = bool(candidate_subject) and candidate_subject == ( + str(rec.subject_key or "").strip() + ) + same_claim = same_subject and candidate_kind == str(rec.claim_kind or "").strip() + if same_claim: + candidate_normalized = " ".join(candidate_text.split()).casefold() + record_normalized = " ".join(rec.content.split()).casefold() + if candidate_normalized == record_normalized: + return Resolution( + ResolutionOp.NOOP, + target_id=rec.id, + reason=f"exact duplicate of keyed claim {rec.id}", + ) + return Resolution(ResolutionOp.INVALIDATE, target_id=rec.id, + reason=f"supersedes {rec.id} (shared claim key, " + f"token overlap={overlap:.2f}, similarity={sim:.2f})") if overlap >= DUP_TOKEN_JACCARD: + if candidate_subject: + return Resolution( + ResolutionOp.INVALIDATE, + target_id=rec.id, + reason=f"replaces unkeyed duplicate {rec.id} with durable claim identity " + f"(token overlap={overlap:.2f})", + ) return Resolution(ResolutionOp.NOOP, target_id=rec.id, reason=f"near-duplicate of {rec.id} (token overlap={overlap:.2f})") - if overlap >= SUBJECT_TOKEN_JACCARD: + # Without an explicit claim key, invalidation needs strong agreement from + # lexical and semantic signals. A high cosine alone can be a topical + # paraphrase rather than a contradiction, so it becomes a relation instead. + if overlap >= STRONG_SUBJECT_TOKEN_JACCARD and sim >= STRONG_JOINT_EMBED_SIM: return Resolution(ResolutionOp.INVALIDATE, target_id=rec.id, - reason=f"supersedes {rec.id} (same subject, " + reason=f"supersedes {rec.id} (strong joint evidence: " f"token overlap={overlap:.2f}, similarity={sim:.2f})") - # Token overlap says "distinct", but a high-enough embedding cosine says "same fact - # in different words" — the paraphrase case token Jaccard cannot see (the known - # ceiling this second signal exists to close). if best_sim is not None and best_sim[0] >= PARAPHRASE_EMBED_SIM: psim, prec = best_sim povl = jaccard(cand_tokens, tokenize(f"{prec.title} {prec.content}")) - return Resolution(ResolutionOp.INVALIDATE, target_id=prec.id, - reason=f"supersedes {prec.id} (paraphrase: cosine={psim:.2f}, " + return Resolution(ResolutionOp.RELATE, target_id=prec.id, + reason=f"related to {prec.id} (paraphrase-like: cosine={psim:.2f}, " f"token overlap={povl:.2f})") + if overlap >= SUBJECT_TOKEN_JACCARD: + return Resolution(ResolutionOp.RELATE, target_id=rec.id, + reason=f"related to {rec.id} (same topic, " + f"token overlap={overlap:.2f}, similarity={sim:.2f})") return Resolution(ResolutionOp.ADD, reason=f"related but distinct (best overlap={overlap:.2f})") diff --git a/engraphis/core/retrieval_policy.py b/engraphis/core/retrieval_policy.py new file mode 100644 index 00000000..dbf93750 --- /dev/null +++ b/engraphis/core/retrieval_policy.py @@ -0,0 +1,92 @@ +"""Deterministic retrieval-profile selection. + +``balanced`` preserves the established hybrid path. ``auto`` is explicit and +conservative: it only selects a specialized profile when the query has a strong, +locally-observable signal. This keeps automatic routing measurable and prevents +an unbenchmarked policy change from silently altering existing callers. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + + +RETRIEVAL_PROFILES = frozenset({"balanced", "auto", "lexical", "graph", "code"}) + +_CODE_RE = re.compile( + r"(?:\w+[./\\])+\w+|::|->|\b(?:class|def|function|import|module)\b|" + r"\b[A-Za-z_]\w*\([^)]*\)", + re.IGNORECASE, +) +_GRAPH_RE = re.compile( + r"\b(?:calls?|causes?|depends?|impact|path|related|relationship|why)\b", + re.IGNORECASE, +) +_LEXICAL_RE = re.compile( + r"\"[^\"]+\"|'[^']+'|\b[A-Z][A-Z0-9_]{2,}\b|" + r"\b(?:exact|identifier|literal|named|spelled)\b", +) + + +@dataclass(frozen=True) +class ProfileConfig: + name: str + vector: bool + lexical: bool + graph: bool + code: bool + semantic_scale: float = 1.0 + lexical_scale: float = 1.0 + graph_scale: float = 1.0 + code_scale: float = 1.0 + graph_presence_bonus: float = 0.0 + code_presence_bonus: float = 0.0 + + +_CONFIGS = { + "balanced": ProfileConfig("balanced", True, True, True, False), + "lexical": ProfileConfig("lexical", False, True, False, False), + # Specialized profiles retain supporting arms but make their declared + # evidence type decisive. ``balanced`` stays byte-for-byte equivalent to + # the established scoring behavior, and ``auto`` remains opt-in. + "graph": ProfileConfig( + "graph", True, True, True, False, + graph_scale=3.0, graph_presence_bonus=1.5, + ), + "code": ProfileConfig( + "code", True, True, True, True, + code_scale=3.0, code_presence_bonus=1.5, + ), +} + + +def profile_config(name: str) -> ProfileConfig: + """Return a validated immutable configuration for a concrete profile.""" + normalized = str(name or "").strip().casefold() + if normalized not in _CONFIGS: + choices = ", ".join(sorted(_CONFIGS)) + raise ValueError(f"retrieval profile must resolve to one of: {choices}") + return _CONFIGS[normalized] + + +class DeterministicRetrievalPolicy: + """Offline automatic profile selector with stable, inspectable rules.""" + + identity = "engraphis.deterministic.v1" + + def profile(self, query: str) -> str: + if _CODE_RE.search(query or ""): + return "code" + if _GRAPH_RE.search(query or ""): + return "graph" + if _LEXICAL_RE.search(query or ""): + return "lexical" + return "balanced" + + def resolve(self, requested: str, query: str) -> ProfileConfig: + normalized = str(requested or "balanced").strip().casefold() + if normalized not in RETRIEVAL_PROFILES: + choices = ", ".join(sorted(RETRIEVAL_PROFILES)) + raise ValueError(f"retrieval_profile must be one of: {choices}") + selected = self.profile(query) if normalized == "auto" else normalized + return profile_config(selected) diff --git a/engraphis/core/schema.py b/engraphis/core/schema.py index 140cd849..0efabdda 100644 --- a/engraphis/core/schema.py +++ b/engraphis/core/schema.py @@ -8,7 +8,7 @@ """ from __future__ import annotations -SCHEMA_VERSION = 4 +SCHEMA_VERSION = 5 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -73,8 +73,11 @@ last_access REAL, valid_from REAL, -- world-time validity valid_to REAL, + valid_to_recorded_at REAL, -- system-time when valid_to was learned ingested_at REAL, -- system-time validity expired_at REAL, + subject_key TEXT DEFAULT '', -- stable claim subject, optional + claim_kind TEXT DEFAULT '', -- optional claim predicate/category pinned INTEGER DEFAULT 0, sensitivity TEXT DEFAULT 'normal', provenance TEXT DEFAULT '{}', @@ -84,6 +87,31 @@ CREATE INDEX IF NOT EXISTS idx_mem_session ON memories(session_id); CREATE INDEX IF NOT EXISTS idx_mem_valid ON memories(valid_from, valid_to, expired_at); +-- Persisted memory↔entity incidence lets graph retrieval attach evidence without +-- rescanning every memory's prose. Temporal fields preserve historical walks. +CREATE TABLE IF NOT EXISTS memory_entities ( + id TEXT PRIMARY KEY, + memory_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + workspace_id TEXT, + repo_id TEXT, + source_kind TEXT NOT NULL DEFAULT 'edge_support', + confidence REAL NOT NULL DEFAULT 1.0, + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL, + provenance TEXT DEFAULT '{}' +); +CREATE INDEX IF NOT EXISTS idx_memory_entity_entity + ON memory_entities(workspace_id, repo_id, entity_id, valid_to, expired_at); +CREATE INDEX IF NOT EXISTS idx_memory_entity_memory + ON memory_entities(memory_id, valid_to, expired_at); +CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_entity_live_unique + ON memory_entities(memory_id, entity_id, source_kind) + WHERE valid_to IS NULL AND expired_at IS NULL; + -- Vectors (Phase 0 reference store; Phase 1 → sqlite-vec vec0 virtual table). CREATE TABLE IF NOT EXISTS mem_vectors ( id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE, @@ -118,6 +146,7 @@ weight REAL DEFAULT 1.0, valid_from REAL, valid_to REAL, + valid_to_recorded_at REAL, ingested_at REAL, expired_at REAL, provenance TEXT DEFAULT '{}' @@ -141,6 +170,7 @@ confidence REAL NOT NULL DEFAULT 0.5, valid_from REAL, valid_to REAL, + valid_to_recorded_at REAL, ingested_at REAL, expired_at REAL, provenance TEXT DEFAULT '{}', @@ -262,7 +292,14 @@ relation TEXT, layer TEXT DEFAULT 'semantic', reason TEXT DEFAULT '', - created_at REAL + created_at REAL, + -- Direct memory relationships participate in graph recall. Give them the + -- same history as other graph bridges so later links cannot alter past reads. + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_mem_links_ab ON mem_links(a, b); -- Links are undirected: Store.get_links()/has_link()/add_link() all match "a=? OR b=?". @@ -284,7 +321,12 @@ exported INTEGER, content_hash TEXT, embedding_ref TEXT, - updated_at REAL + updated_at REAL, + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_sym_repo ON symbols(repo_id, name); @@ -296,7 +338,12 @@ relation TEXT, -- calls|imports|references|implements|tests layer TEXT DEFAULT 'entity', file TEXT, - line INTEGER + line INTEGER, + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_code_edge_src ON code_edges(repo_id, src); CREATE INDEX IF NOT EXISTS idx_code_edge_dst ON code_edges(repo_id, dst); @@ -322,7 +369,11 @@ relation TEXT DEFAULT 'mentions', confidence REAL DEFAULT 1.0, created_at REAL, - UNIQUE(repo_id, symbol_id, memory_id, relation) + valid_from REAL, + valid_to REAL, + valid_to_recorded_at REAL, + ingested_at REAL, + expired_at REAL ); CREATE INDEX IF NOT EXISTS idx_code_mem_symbol ON code_memory_links(repo_id, symbol_id); @@ -363,6 +414,7 @@ operation TEXT NOT NULL, workspace_id TEXT, repo_id TEXT, + sequence INTEGER NOT NULL CHECK(sequence >= 1), scope_digest TEXT NOT NULL, actor TEXT DEFAULT 'system', target_count INTEGER DEFAULT 0, diff --git a/engraphis/core/store.py b/engraphis/core/store.py index d868569d..4058b50a 100644 --- a/engraphis/core/store.py +++ b/engraphis/core/store.py @@ -12,6 +12,7 @@ import hashlib import json +import math import os import re import sqlite3 @@ -189,6 +190,28 @@ def _edge_support_confidence(provenance: Any, source_kind: str) -> float: return _SUPPORT_CONFIDENCE.get(source_kind, 0.50) +_PUBLIC_RECEIPT_LABELS_BY_KEY = { + "mtype": {"working", "episodic", "semantic", "procedural"}, + "scope": {"session", "repo", "workspace", "user"}, + "resolution": {"add", "noop", "invalidate", "relate"}, + "retention": { + "ephemeral", "normal", "critical", "short", "standard", "long", "permanent", + }, + "intent": { + "recall", "recall_context", "grounded", "http_read_only", + "explain", "timeline", "code", "locate_code", + }, + "relation": { + "related", "mentions", "supports", "supersedes", "consolidates", + "promotes", "causes", "depends_on", "calls", "imports", "references", + "implements", "tests", "uses", "owned_by", "co_occurs", + }, + "layer": {"temporal", "entity", "causal", "semantic"}, + "retrieval_profile": {"balanced", "auto", "lexical", "graph", "code"}, + "response_mode": {"full", "compact"}, +} + + def _receipt_metadata(metadata: dict) -> dict: """Keep receipt metadata useful but content-free and bounded.""" allowed = { @@ -197,27 +220,210 @@ def _receipt_metadata(metadata: dict) -> dict: "files_scanned", "files_indexed", "files_removed", "symbols", "edges", "entities", "relations", "tables", "dry_run", "error_count", "entities_added", "relations_added", + "retrieval_profile", "response_mode", "historical", "token_usage", } + def content_free_label(key: str, value: str) -> str: + normalized = value.strip().casefold().replace(" ", "_") + if normalized in _PUBLIC_RECEIPT_LABELS_BY_KEY.get(key, set()): + return normalized + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + out: dict[str, Any] = {} for key in sorted(metadata, key=lambda item: str(item))[:24]: safe_key = str(key)[:64] if safe_key not in allowed: continue value = metadata[key] - if isinstance(value, bool) or value is None: + if safe_key == "token_usage": + if not isinstance(value, dict): + continue + numeric = { + name: value[name] + for name in ( + "budget_tokens", "context_tokens", "source_tokens", "saved_tokens", + "savings_ratio", "packed_count", "omitted_count", + ) + if type(value.get(name)) in (int, float) + and math.isfinite(float(value[name])) + } + counter = value.get("token_counter") + if isinstance(counter, str): + if counter in {"engraphis.regex.v1", "estimate_tokens"}: + numeric["token_counter"] = counter + else: + numeric["token_counter"] = ( + "sha256:" + hashlib.sha256(counter.encode("utf-8")).hexdigest() + ) + out[safe_key] = numeric + elif isinstance(value, bool) or value is None: out[safe_key] = value elif isinstance(value, (int, float)): - out[safe_key] = value + if math.isfinite(float(value)): + out[safe_key] = value elif isinstance(value, str): - out[safe_key] = ( - value[:80] if len(value) <= 80 - else "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() - ) + out[safe_key] = content_free_label(safe_key, value) elif isinstance(value, (list, tuple)): out[safe_key] = len(value) return out +_PUBLIC_RECEIPT_ID = re.compile(r"^rcpt_[0-9ABCDEFGHJKMNPQRSTVWXYZ]{26}$") +_PUBLIC_RECEIPT_HASH = re.compile(r"^[0-9a-f]{64}$") +_PUBLIC_RECEIPT_HASHED_LABEL = re.compile(r"^sha256:[0-9a-f]{64}$") +_PUBLIC_RECEIPT_KEYS = { + "version", "id", "ts_ms", "operation", "scope_digest", "actor_digest", + "target_count", "status", "metadata", "prev_hash", +} +_PUBLIC_RECEIPT_METADATA_KEYS = { + "mtype", "scope", "resolution", "retention", "extracted", "intent", "k", + "result_count", "grounded", "citations", "relation", "layer", "graph_layers", + "files_scanned", "files_indexed", "files_removed", "symbols", "edges", + "entities", "relations", "tables", "dry_run", "error_count", + "entities_added", "relations_added", "retrieval_profile", "response_mode", + "historical", "token_usage", +} +_PUBLIC_RECEIPT_OPERATIONS = { + "remember", "recall", "promote", "link", "index_repo", + "graph_index", "grounded_recall", "consolidate", "sync", +} +_PUBLIC_RECEIPT_STATUSES = { + "ok", "add", "noop", "invalidate", "relate", "ingested", + "postgres_schema", "grounded", "abstained", "promoted", + "indexed", "skipped", "error", "failed", "cancelled", "partial", +} + + +def _redacted_receipt_value(value: Any) -> str: + raw = value if isinstance(value, str) else str(value or "") + return "redacted_sha256:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _public_receipt_row(row: dict) -> dict: + """Return one validated content-free receipt or a hash-only corruption marker.""" + raw = row.get("payload") + raw = raw if isinstance(raw, str) else str(raw or "") + raw_id = row.get("id") + raw_prev = row.get("prev_hash") + raw_hash = row.get("receipt_hash") + + def safe_id() -> str: + value = raw_id if isinstance(raw_id, str) else str(raw_id or "") + return value if _PUBLIC_RECEIPT_ID.fullmatch(value) else _redacted_receipt_value(value) + + def safe_hash(value: Any, *, allow_empty: bool = False) -> str: + text = value if isinstance(value, str) else str(value or "") + if allow_empty and not text: + return "" + return ( + text if _PUBLIC_RECEIPT_HASH.fullmatch(text) + else _redacted_receipt_value(text) + ) + + invalid = { + "id": safe_id(), + "prev_hash": safe_hash(raw_prev, allow_empty=True), + "hash": safe_hash(raw_hash), + "invalid_payload": True, + "payload_bytes": len(raw.encode("utf-8")), + "payload_sha256": hashlib.sha256(raw.encode("utf-8")).hexdigest(), + } + try: + payload = json.loads(raw) + except (TypeError, ValueError, RecursionError): + return invalid + if ( + not isinstance(payload, dict) + or set(payload) != _PUBLIC_RECEIPT_KEYS + or payload.get("version") != 1 + or payload.get("id") != raw_id + or payload.get("prev_hash") != raw_prev + or not isinstance(raw_id, str) + or _PUBLIC_RECEIPT_ID.fullmatch(raw_id) is None + or ( + raw_prev != "" + and ( + not isinstance(raw_prev, str) + or _PUBLIC_RECEIPT_HASH.fullmatch(raw_prev) is None + ) + ) + or not isinstance(raw_hash, str) + or _PUBLIC_RECEIPT_HASH.fullmatch(raw_hash) is None + or hashlib.sha256(raw.encode("utf-8")).hexdigest() != raw_hash + ): + return invalid + if type(payload.get("ts_ms")) is not int or payload["ts_ms"] < 0: + return invalid + if type(payload.get("target_count")) is not int or payload["target_count"] < 0: + return invalid + operation = payload.get("operation") + if not ( + operation in _PUBLIC_RECEIPT_OPERATIONS + or ( + isinstance(operation, str) + and _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(operation) + ) + ): + return invalid + status = payload.get("status") + if not ( + status in _PUBLIC_RECEIPT_STATUSES + or ( + isinstance(status, str) + and _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(status) + ) + ): + return invalid + if not ( + isinstance(payload.get("scope_digest"), str) + and re.fullmatch(r"[0-9a-f]{24}", payload["scope_digest"]) + and isinstance(payload.get("actor_digest"), str) + and re.fullmatch(r"[0-9a-f]{16}", payload["actor_digest"]) + ): + return invalid + metadata = payload.get("metadata") + if not isinstance(metadata, dict) or not set(metadata).issubset( + _PUBLIC_RECEIPT_METADATA_KEYS + ): + return invalid + for key, value in metadata.items(): + if key == "token_usage": + if not isinstance(value, dict): + return invalid + allowed_usage = { + "budget_tokens", "context_tokens", "source_tokens", "saved_tokens", + "savings_ratio", "packed_count", "omitted_count", "token_counter", + } + if not set(value).issubset(allowed_usage): + return invalid + for usage_key, usage_value in value.items(): + if usage_key == "token_counter": + if not ( + usage_value in {"engraphis.regex.v1", "estimate_tokens"} + or ( + isinstance(usage_value, str) + and _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(usage_value) + ) + ): + return invalid + elif ( + type(usage_value) not in (int, float) + or not math.isfinite(float(usage_value)) + ): + return invalid + elif isinstance(value, str): + public_labels = _PUBLIC_RECEIPT_LABELS_BY_KEY.get(key, set()) + if not ( + value in public_labels + or _PUBLIC_RECEIPT_HASHED_LABEL.fullmatch(value) + ): + return invalid + elif isinstance(value, bool) or value is None: + continue + elif type(value) not in (int, float) or not math.isfinite(float(value)): + return invalid + return {**payload, "hash": raw_hash} + + def _fts5_available(conn: sqlite3.Connection) -> bool: try: conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts_probe USING fts5(x)") @@ -227,6 +433,39 @@ def _fts5_available(conn: sqlite3.Connection) -> bool: return False +def _temporal_anchors(flt: Optional[SearchFilter], *, valid_at: Optional[float] = None + ) -> tuple[float, float]: + """Return world-time and system-time anchors for one read. + + ``valid_at`` is an explicit per-operation override used by graph traversal; + otherwise the filter's normalized ``valid_at``/legacy ``as_of`` value applies. + System-time defaults to the present, which preserves ordinary current reads. + """ + world = valid_at + if world is None and flt is not None: + world = flt.valid_at + known = flt.known_at if flt is not None else None + present = now_ts() + return (present if world is None else world, + present if known is None else known) + + +def _temporal_visibility_sql(alias: str, flt: Optional[SearchFilter], *, + valid_at: Optional[float] = None) -> tuple[str, list[Any]]: + """SQL predicate shared by temporal code-history reads.""" + world, known = _temporal_anchors(flt, valid_at=valid_at) + p = f"{alias}." if alias else "" + return ( + f"({p}valid_from IS NULL OR {p}valid_from<=?) " + f"AND ({p}valid_to IS NULL OR ?<{p}valid_to " + f"OR ({p}valid_to_recorded_at IS NOT NULL " + f"AND ?<{p}valid_to_recorded_at)) " + f"AND ({p}ingested_at IS NULL OR {p}ingested_at<=?) " + f"AND ({p}expired_at IS NULL OR ?<{p}expired_at)", + [world, world, known, known, known], + ) + + def memory_matches_filter(rec: MemoryRecord, flt: Optional[SearchFilter], *, at: Optional[float] = None, include_invalid: bool = False) -> bool: @@ -270,14 +509,18 @@ def memory_matches_filter(rec: MemoryRecord, flt: Optional[SearchFilter], *, return False if include_invalid: return True - t = at if at is not None else ( - flt.as_of if flt and flt.as_of is not None else now_ts() - ) - if rec.expired_at is not None: + valid_at, known_at = _temporal_anchors(flt, valid_at=at) + if rec.ingested_at is not None and rec.ingested_at > known_at: + return False + if rec.expired_at is not None and known_at >= rec.expired_at: return False - if rec.valid_from is not None and rec.valid_from > t: + if rec.valid_from is not None and rec.valid_from > valid_at: return False - if rec.valid_to is not None and t >= rec.valid_to: + if (rec.valid_to is not None and valid_at >= rec.valid_to + and not ( + rec.valid_to_recorded_at is not None + and known_at < rec.valid_to_recorded_at + )): return False return True @@ -559,18 +802,25 @@ def _cleanup_v4_backup_temps(self, backup_path: str) -> None: if changed: self._fsync_backup_parent(backup_path) - def _backup_before_v4_migration(self) -> str: - """Create and verify the mandatory pre-v4 backup without mutating source data. + def _backup_before_v4_migration(self, *, previous_version: int = 0) -> str: + """Create and verify the mandatory pre-migration backup without mutating data. Source and destination both use the injected connector, so SQLCipher databases remain keyed throughout. The caller holds ``BEGIN IMMEDIATE`` on the primary connection, preventing another writer from changing the source between this snapshot and the migration commit. Only a quick-checked temporary backup may atomically replace the stable backup path; every failure aborts the migration. + + The v3→v4 release established ``.pre-migration-v4.bak`` as a durable recovery + artifact. A database that subsequently upgrades from v4 to v5 can legitimately + still have that older snapshot. Reusing the same filename would compare the v3 + snapshot with the current v4 source and abort every v5 upgrade. Preserve the + legacy name for pre-v4 sources and use a version-specific v5 name for v4. """ if self.path in (":memory:", "") or self.path.startswith("file::memory:"): - raise RuntimeError("schema v4 migration requires a durable pre-migration backup") - backup_path = f"{self.path}.pre-migration-v4.bak" + raise RuntimeError("schema migration requires a durable pre-migration backup") + backup_version = 5 if previous_version >= 4 else 4 + backup_path = f"{self.path}.pre-migration-v{backup_version}.bak" self._cleanup_v4_backup_temps(backup_path) temp_path = ( f"{backup_path}.tmp-{os.getpid()}-{threading.get_ident()}-{time.time_ns()}" @@ -657,7 +907,7 @@ def _backup_before_v4_migration(self) -> str: except OSError: pass raise RuntimeError( - "schema v4 migration aborted: could not create and verify the " + f"schema v{backup_version} migration aborted: could not create and verify the " "pre-migration backup" ) from exc @@ -692,18 +942,33 @@ def init_schema(self) -> None: ).fetchone() value = row[0] if row is not None else None previous_version = int(value) if value is not None else 0 + # Early v5 databases recorded direct memory links with only ``created_at``. + # Track that shape independently of the version row so they receive the + # missing bi-temporal fields and backfill on their next safe open. + mem_link_columns: set[str] = set() + if "mem_links" in object_names: + mem_link_columns = { + str(row["name"]) + for row in self.conn.execute("PRAGMA table_info(mem_links)").fetchall() + } + mem_links_need_temporal_backfill = not { + "valid_from", "valid_to", "valid_to_recorded_at", "ingested_at", "expired_at", + }.issubset(mem_link_columns) + self._mem_links_need_temporal_backfill = mem_links_need_temporal_backfill if previous_version > SCHEMA_VERSION: raise RuntimeError( f"database schema {previous_version} is newer than supported " f"schema {SCHEMA_VERSION}" ) - needs_backup = bool(object_names) and previous_version < SCHEMA_VERSION + needs_backup = bool(object_names) and ( + previous_version < SCHEMA_VERSION or mem_links_need_temporal_backfill + ) try: # Reserve the writer before the snapshot. This is read/locking state only; # every schema/data transform remains inside the transaction below. self.conn.execute("BEGIN IMMEDIATE") if needs_backup: - self._backup_before_v4_migration() + self._backup_before_v4_migration(previous_version=previous_version) self._apply_schema(previous_version) self.conn.commit() except BaseException: @@ -712,6 +977,15 @@ def init_schema(self) -> None: raise def _apply_schema(self, previous_version: int) -> None: + mem_links_need_temporal_backfill = bool( + getattr(self, "_mem_links_need_temporal_backfill", False) + ) + receipt_sequence_existed = any( + str(row["name"]) == "sequence" + for row in self.conn.execute( + "PRAGMA table_info(operation_receipts)" + ).fetchall() + ) self._execute_script_transactional(SCHEMA_SQL) self.has_fts5 = _fts5_available(self.conn) self.conn.execute(FTS_SQL_FTS5 if self.has_fts5 else FTS_SQL_FALLBACK) @@ -720,15 +994,38 @@ def _apply_schema(self, previous_version: int) -> None: # explicit, idempotent ALTER TABLE here (SQLite has no "ADD COLUMN IF NOT EXISTS"). for stmt in ( "ALTER TABLE memories ADD COLUMN sort_order REAL", + "ALTER TABLE memories ADD COLUMN subject_key TEXT DEFAULT ''", + "ALTER TABLE memories ADD COLUMN claim_kind TEXT DEFAULT ''", + "ALTER TABLE memories ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE edges ADD COLUMN layer TEXT DEFAULT 'semantic'", "ALTER TABLE entities ADD COLUMN normalized_name TEXT NOT NULL DEFAULT ''", "ALTER TABLE entities ADD COLUMN canonical_method TEXT NOT NULL DEFAULT 'exact'", "ALTER TABLE entities ADD COLUMN canonical_confidence REAL NOT NULL DEFAULT 1.0", "ALTER TABLE mem_links ADD COLUMN layer TEXT DEFAULT 'semantic'", "ALTER TABLE mem_links ADD COLUMN reason TEXT DEFAULT ''", + "ALTER TABLE mem_links ADD COLUMN valid_from REAL", + "ALTER TABLE mem_links ADD COLUMN valid_to REAL", + "ALTER TABLE mem_links ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE mem_links ADD COLUMN ingested_at REAL", + "ALTER TABLE mem_links ADD COLUMN expired_at REAL", "ALTER TABLE code_edges ADD COLUMN layer TEXT DEFAULT 'entity'", "ALTER TABLE symbols ADD COLUMN docstring TEXT DEFAULT ''", + "ALTER TABLE symbols ADD COLUMN valid_from REAL", + "ALTER TABLE symbols ADD COLUMN valid_to REAL", + "ALTER TABLE symbols ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE symbols ADD COLUMN ingested_at REAL", + "ALTER TABLE symbols ADD COLUMN expired_at REAL", + "ALTER TABLE code_edges ADD COLUMN valid_from REAL", + "ALTER TABLE code_edges ADD COLUMN valid_to REAL", + "ALTER TABLE code_edges ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE code_edges ADD COLUMN ingested_at REAL", + "ALTER TABLE code_edges ADD COLUMN expired_at REAL", + "ALTER TABLE edges ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE edge_supports ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE memory_entities ADD COLUMN valid_to_recorded_at REAL", + "ALTER TABLE code_memory_links ADD COLUMN valid_to_recorded_at REAL", "ALTER TABLE receipt_chain_heads ADD COLUMN integrity_error TEXT DEFAULT ''", + "ALTER TABLE operation_receipts ADD COLUMN sequence INTEGER", "ALTER TABLE jobs ADD COLUMN runner_id TEXT", "ALTER TABLE jobs ADD COLUMN heartbeat_at REAL", ): @@ -736,6 +1033,48 @@ def _apply_schema(self, previous_version: int) -> None: self.conn.execute(stmt) except sqlite3.OperationalError: pass # column already exists + # This cannot live in SCHEMA_SQL: CREATE TABLE IF NOT EXISTS leaves an + # early-v5 ``mem_links`` table untouched, so the index would reference + # temporal columns before the additive ALTERs above install them. + self.conn.execute( + "CREATE INDEX IF NOT EXISTS idx_mem_links_temporal " + "ON mem_links(a, valid_to, expired_at)" + ) + self.conn.execute( + "UPDATE operation_receipts SET workspace_id='' WHERE workspace_id IS NULL" + ) + self.conn.execute( + "UPDATE operation_receipts SET repo_id='' WHERE repo_id IS NULL" + ) + if not receipt_sequence_existed: + self._backfill_receipt_sequences() + self._execute_script_transactional( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_receipt_sequence " + "ON operation_receipts(workspace_id, sequence) " + "WHERE sequence IS NOT NULL;" + "DROP TRIGGER IF EXISTS trg_receipt_sequence_required;" + "CREATE TRIGGER trg_receipt_sequence_required " + "BEFORE INSERT ON operation_receipts " + "WHEN NEW.sequence IS NULL OR typeof(NEW.sequence)!='integer' " + "OR NEW.sequence<1 BEGIN " + "SELECT RAISE(ABORT, 'receipt sequence is required'); END;" + "DROP TRIGGER IF EXISTS trg_receipt_sequence_immutable;" + "CREATE TRIGGER trg_receipt_sequence_immutable " + "BEFORE UPDATE OF sequence ON operation_receipts " + "WHEN NEW.sequence IS NOT OLD.sequence BEGIN " + "SELECT RAISE(ABORT, 'receipt sequence is immutable'); END;" + ) + # These are migration transforms, not startup maintenance. Re-running the + # incidence backfill on every open scans the entire evidence graph and turns + # otherwise constant-time startup into O(workspace history). The schema-version + # row is written in the same transaction below, so an interrupted migration + # remains < v5 and safely retries all three transforms. + if previous_version < 5: + self._migrate_code_history_v5() + self._backfill_claim_identity_v5() + self._backfill_memory_entities_v5() + if previous_version < 5 or mem_links_need_temporal_backfill: + self._migrate_mem_link_history_v5() # Classify pre-v3 edges. Existing rows defaulted to semantic during ALTER TABLE; # infer their more specific logical layer from the relationship label. if previous_version < 3: @@ -780,6 +1119,24 @@ def _apply_schema(self, previous_version: int) -> None: "WHERE workspace_id IS NOT NULL AND repo_id IS NOT NULL " "AND valid_to IS NULL AND expired_at IS NULL;" ) + self._execute_script_transactional( + "CREATE INDEX IF NOT EXISTS idx_mem_claim_live " + "ON memories(workspace_id, repo_id, scope, mtype, subject_key, claim_kind) " + "WHERE subject_key<>'' AND valid_to IS NULL AND expired_at IS NULL;" + "CREATE INDEX IF NOT EXISTS idx_sym_repo_live " + "ON symbols(repo_id, file, fqname, valid_to, expired_at);" + "CREATE INDEX IF NOT EXISTS idx_code_edge_live " + "ON code_edges(repo_id, file, valid_to, expired_at);" + "CREATE UNIQUE INDEX IF NOT EXISTS idx_code_mem_live_unique " + "ON code_memory_links(repo_id, symbol_id, memory_id, relation) " + "WHERE valid_to IS NULL AND expired_at IS NULL;" + "CREATE INDEX IF NOT EXISTS idx_code_mem_symbol " + "ON code_memory_links(repo_id, symbol_id);" + "CREATE INDEX IF NOT EXISTS idx_code_mem_memory " + "ON code_memory_links(repo_id, memory_id);" + "CREATE INDEX IF NOT EXISTS idx_code_mem_live_symbol " + "ON code_memory_links(repo_id, symbol_id, valid_to, expired_at);" + ) # Every workspace has a cheap graph generation/state row, including databases # that already contained graph data before the v4 explorer tables were added. # Triggers in SCHEMA_SQL advance the generation on subsequent graph mutations. @@ -793,28 +1150,196 @@ def _apply_schema(self, previous_version: int) -> None: # anchor table existed. From this point onward every append updates it atomically, # allowing verification to detect deletion of the newest receipt as well as an # interior chain break. + if previous_version < 5: + receipt_scopes = self.conn.execute( + "SELECT r.workspace_id, COALESCE(MAX(r.ts), 0) AS updated_at " + "FROM operation_receipts r " + "LEFT JOIN receipt_chain_heads h ON h.workspace_id=r.workspace_id " + "WHERE h.workspace_id IS NULL " + "GROUP BY r.workspace_id" + ).fetchall() + for receipt_scope in receipt_scopes: + workspace_id = str(receipt_scope["workspace_id"] or "") + chain = self._receipt_chain_state(workspace_id) + self.conn.execute( + "INSERT OR IGNORE INTO receipt_chain_heads " + "(workspace_id, receipt_count, head_hash, integrity_error, updated_at) " + "VALUES (?,?,?,?,?)", + ( + workspace_id, + len(chain["rows"]), + chain["head"], + "" if not chain["errors"] else "migration_chain_invalid", + receipt_scope["updated_at"], + ), + ) self.conn.execute( - "UPDATE operation_receipts SET workspace_id='' WHERE workspace_id IS NULL" + "INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES (?,?)", + (SCHEMA_VERSION, now_ts()), ) + + def _migrate_code_history_v5(self) -> None: + """Give pre-v5 code graph rows open bi-temporal intervals. + + ``code_memory_links`` formerly had a table-level uniqueness constraint, which + made it impossible to retain a closed link and later create the same live link. + SQLite cannot drop that constraint in place, so rebuild that one narrow table + transactionally before installing the partial live-uniqueness index. + """ + stamp = now_ts() self.conn.execute( - "UPDATE operation_receipts SET repo_id='' WHERE repo_id IS NULL" + "UPDATE symbols SET valid_from=COALESCE(valid_from, updated_at, ?), " + "ingested_at=COALESCE(ingested_at, updated_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), ) self.conn.execute( - "INSERT OR IGNORE INTO receipt_chain_heads " - "(workspace_id, receipt_count, head_hash, integrity_error, updated_at) " - "SELECT COALESCE(r.workspace_id, ''), COUNT(*), " - " (SELECT r2.receipt_hash FROM operation_receipts r2 " - " WHERE COALESCE(r2.workspace_id, '')=COALESCE(r.workspace_id, '') " - " ORDER BY r2.rowid DESC LIMIT 1), " - " '', " - " COALESCE(MAX(r.ts), 0) " - "FROM operation_receipts r GROUP BY COALESCE(r.workspace_id, '')" + "UPDATE code_edges SET valid_from=COALESCE(valid_from, ?), " + "ingested_at=COALESCE(ingested_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), ) + columns = { + row["name"] for row in self.conn.execute( + "PRAGMA table_info(code_memory_links)" + ).fetchall() + } + if "valid_from" not in columns: + self.conn.execute( + "CREATE TABLE code_memory_links_v5 (" + "id TEXT PRIMARY KEY, repo_id TEXT NOT NULL, symbol_id TEXT NOT NULL, " + "memory_id TEXT NOT NULL, relation TEXT DEFAULT 'mentions', " + "confidence REAL DEFAULT 1.0, created_at REAL, valid_from REAL, " + "valid_to REAL, valid_to_recorded_at REAL, " + "ingested_at REAL, expired_at REAL)" + ) + self.conn.execute( + "INSERT INTO code_memory_links_v5(" + "id, repo_id, symbol_id, memory_id, relation, confidence, created_at, " + "valid_from, ingested_at) " + "SELECT id, repo_id, symbol_id, memory_id, relation, confidence, " + "created_at, COALESCE(created_at, ?), COALESCE(created_at, ?) " + "FROM code_memory_links", + (stamp, stamp), + ) + self.conn.execute("DROP TABLE code_memory_links") + self.conn.execute("ALTER TABLE code_memory_links_v5 RENAME TO code_memory_links") + else: + self.conn.execute( + "UPDATE code_memory_links SET valid_from=COALESCE(valid_from, created_at, ?), " + "ingested_at=COALESCE(ingested_at, created_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), + ) + + def _migrate_mem_link_history_v5(self) -> None: + """Give legacy direct memory links an open bi-temporal interval. + + ``created_at`` was the only historical signal on old rows, so it is both + the best available world-time and system-time start. Rows without a clock + start at migration time rather than being projected into every past view. + """ + stamp = now_ts() self.conn.execute( - "INSERT OR IGNORE INTO schema_migrations(version, applied_at) VALUES (?,?)", - (SCHEMA_VERSION, now_ts()), + "UPDATE mem_links SET valid_from=COALESCE(valid_from, created_at, ?), " + "ingested_at=COALESCE(ingested_at, created_at, ?) " + "WHERE valid_from IS NULL OR ingested_at IS NULL", + (stamp, stamp), ) + def _backfill_claim_identity_v5(self) -> None: + """Lift already-present metadata hints into indexed, optional claim columns.""" + rows = self.conn.execute( + "SELECT id, metadata, subject_key, claim_kind FROM memories" + ).fetchall() + for row in rows: + metadata = _loads(row["metadata"], {}) + if not isinstance(metadata, dict): + metadata = {} + subject_key = str(row["subject_key"] or metadata.get("subject_key") or "").strip() + claim_kind = str(row["claim_kind"] or metadata.get("claim_kind") or "").strip() + if subject_key != (row["subject_key"] or "") or claim_kind != (row["claim_kind"] or ""): + self.conn.execute( + "UPDATE memories SET subject_key=?, claim_kind=? WHERE id=?", + (subject_key, claim_kind, row["id"]), + ) + + def _backfill_memory_entities_v5(self, memory_id: Optional[str] = None) -> None: + """Materialize deterministic incidence already evidenced by graph supports.""" + sql = ( + "SELECT s.memory_id, endpoint.entity_id, e.workspace_id, e.repo_id, " + "s.confidence, s.valid_from, s.valid_to, s.valid_to_recorded_at, " + "s.ingested_at, s.expired_at, " + "e.valid_from AS edge_valid_from, e.valid_to AS edge_valid_to, " + "e.valid_to_recorded_at AS edge_valid_to_recorded_at, " + "e.ingested_at AS edge_ingested_at, e.expired_at AS edge_expired_at, " + "s.provenance " + "FROM edge_supports s JOIN edges e ON e.id=s.edge_id " + "JOIN (SELECT id AS edge_id, src AS entity_id FROM edges " + "UNION ALL SELECT id, dst FROM edges) endpoint ON endpoint.edge_id=e.id " + "WHERE 1=1" + ) + params: list[Any] = [] + if memory_id is not None: + sql += " AND s.memory_id=?" + params.append(memory_id) + rows = self.conn.execute(sql, params).fetchall() + for row in rows: + valid_starts = [ + value for value in (row["valid_from"], row["edge_valid_from"]) + if value is not None + ] + valid_ends = [ + value for value in (row["valid_to"], row["edge_valid_to"]) + if value is not None + ] + known_starts = [ + value for value in (row["ingested_at"], row["edge_ingested_at"]) + if value is not None + ] + known_ends = [ + value for value in (row["expired_at"], row["edge_expired_at"]) + if value is not None + ] + valid_from = max(valid_starts) if valid_starts else None + valid_to = min(valid_ends) if valid_ends else None + closure_candidates = [ + (row["valid_to"], row["valid_to_recorded_at"]), + (row["edge_valid_to"], row["edge_valid_to_recorded_at"]), + ] + controlling_closures = [ + recorded for end, recorded in closure_candidates + if end is not None and end == valid_to + ] + valid_to_recorded_at = ( + None + if not controlling_closures or any( + recorded is None for recorded in controlling_closures + ) + else min(controlling_closures) + ) + ingested_at = max(known_starts) if known_starts else None + expired_at = min(known_ends) if known_ends else None + if (valid_from is not None and valid_to is not None + and valid_from >= valid_to): + continue + if (ingested_at is not None and expired_at is not None + and ingested_at >= expired_at): + continue + self.link_memory_entity( + memory_id=row["memory_id"], entity_id=row["entity_id"], + workspace_id=row["workspace_id"], repo_id=row["repo_id"], + source_kind="edge_support", confidence=row["confidence"], + valid_from=valid_from, valid_to=valid_to, + valid_to_recorded_at=valid_to_recorded_at, + ingested_at=ingested_at, expired_at=expired_at, + provenance=_loads(row["provenance"], {}), commit=False, + ) + + def backfill_memory_entities_for_memory(self, memory_id: str) -> None: + """Materialize the evidence incidence for one freshly written memory.""" + self._backfill_memory_entities_v5(memory_id) + def _backfill_entity_canonicalization(self) -> None: rows = [dict(row) for row in self.conn.execute( "SELECT id, workspace_id, name, etype, canonical_id, normalized_name, " @@ -1007,14 +1532,16 @@ def _deduplicate_live_edges(self) -> None: provenance = {} provenance["canonical_deduplicated_into"] = survivor["id"] self.conn.execute( - "UPDATE edges SET valid_to=?, provenance=? WHERE id=?", - (closed_at, _dumps(provenance), row["id"]), + "UPDATE edges SET valid_to=?, valid_to_recorded_at=?, " + "provenance=? WHERE id=?", + (closed_at, closed_at, _dumps(provenance), row["id"]), ) retired_marks = ",".join("?" for _ in retired_ids) self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id IN (" + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id IN (" + retired_marks + ") AND valid_to IS NULL AND expired_at IS NULL", - (closed_at, *retired_ids), + (closed_at, closed_at, *retired_ids), ) # Retire duplicates before normalizing the survivor endpoints. A pre-release # v4 database may already have the partial unique index; reversing the @@ -1305,15 +1832,17 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, f"existing provenance={existing['provenance']}, " f"incoming provenance={_dumps(rec.provenance)}", commit=False) ts = now_ts() - rec.ingested_at = rec.ingested_at or ts + rec.ingested_at = rec.ingested_at if rec.ingested_at is not None else ts rec.valid_from = rec.valid_from if rec.valid_from is not None else ts - rec.last_access = rec.last_access or ts + rec.last_access = rec.last_access if rec.last_access is not None else ts self.conn.execute( """INSERT INTO memories (id, workspace_id, repo_id, session_id, scope, mtype, title, content, summary, keywords, metadata, importance, surprise, stability, access_count, last_access, - valid_from, valid_to, ingested_at, expired_at, pinned, sensitivity, provenance) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, + subject_key, claim_kind, + pinned, sensitivity, provenance) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET workspace_id=excluded.workspace_id, repo_id=excluded.repo_id, session_id=excluded.session_id, scope=excluded.scope, mtype=excluded.mtype, @@ -1322,14 +1851,19 @@ def add_memory(self, rec: MemoryRecord, *, audit: bool = True, importance=excluded.importance, surprise=excluded.surprise, stability=excluded.stability, access_count=excluded.access_count, last_access=excluded.last_access, valid_from=excluded.valid_from, - valid_to=excluded.valid_to, ingested_at=excluded.ingested_at, - expired_at=excluded.expired_at, pinned=excluded.pinned, + valid_to=excluded.valid_to, + valid_to_recorded_at=excluded.valid_to_recorded_at, + ingested_at=excluded.ingested_at, + expired_at=excluded.expired_at, subject_key=excluded.subject_key, + claim_kind=excluded.claim_kind, pinned=excluded.pinned, sensitivity=excluded.sensitivity, provenance=excluded.provenance""", (rec.id, rec.workspace_id, rec.repo_id, rec.session_id, _enum(rec.scope), _enum(rec.mtype), rec.title, rec.content, rec.summary, _dumps(rec.keywords), _dumps(rec.metadata), rec.importance, rec.surprise, rec.stability, rec.access_count, rec.last_access, rec.valid_from, rec.valid_to, - rec.ingested_at, rec.expired_at, int(rec.pinned), rec.sensitivity, + rec.valid_to_recorded_at, rec.ingested_at, rec.expired_at, + rec.subject_key, rec.claim_kind, + int(rec.pinned), rec.sensitivity, _dumps(rec.provenance)), ) # full-text mirror @@ -1402,9 +1936,13 @@ def list_memories_page(self, flt: Optional[SearchFilter] = None, *, def close_validity(self, memory_id: str, *, at: Optional[float] = None, actor: str = "system", reason: str = "contradicted") -> None: """Bi-temporal invalidation (§8.3): close a fact's validity window without deleting.""" - at = at if at is not None else now_ts() - self.conn.execute("UPDATE memories SET valid_to=? WHERE id=? AND valid_to IS NULL", - (at, memory_id)) + recorded_at = now_ts() + at = at if at is not None else recorded_at + self.conn.execute( + "UPDATE memories SET valid_to=?, valid_to_recorded_at=? " + "WHERE id=? AND valid_to IS NULL", + (at, recorded_at, memory_id), + ) self.audit(actor, "invalidate", memory_id, reason, commit=False) self.invalidate_edges_for_memory(memory_id, at=at, commit=False) self.conn.commit() @@ -1570,6 +2108,131 @@ def list_entities(self, flt: Optional[SearchFilter] = None, workspace_id=r["workspace_id"], repo_id=r["repo_id"], canonical_id=r["canonical_id"]) for r in rows] + def link_memory_entity(self, *, memory_id: str, entity_id: str, + workspace_id: Optional[str], repo_id: Optional[str], + source_kind: str = "explicit", confidence: float = 1.0, + valid_from: Optional[float] = None, + valid_to: Optional[float] = None, + valid_to_recorded_at: Optional[float] = None, + ingested_at: Optional[float] = None, + expired_at: Optional[float] = None, + provenance: Optional[dict] = None, + commit: bool = True) -> str: + """Create one idempotent, live memory↔entity incidence record.""" + if valid_to is None and expired_at is None: + existing = self.conn.execute( + "SELECT id, confidence, valid_from, ingested_at " + "FROM memory_entities WHERE memory_id=? AND entity_id=? " + "AND source_kind=? AND valid_to IS NULL AND expired_at IS NULL", + (memory_id, entity_id, source_kind), + ).fetchone() + else: + existing = self.conn.execute( + "SELECT id FROM memory_entities WHERE memory_id=? AND entity_id=? " + "AND source_kind=? AND valid_from IS ? AND valid_to IS ? " + "AND valid_to_recorded_at IS ? " + "AND ingested_at IS ? AND expired_at IS ?", + ( + memory_id, entity_id, source_kind, valid_from, valid_to, + valid_to_recorded_at, ingested_at, expired_at, + ), + ).fetchone() + if existing is not None: + if valid_to is None and expired_at is None: + valid_values = [ + value for value in (existing["valid_from"], valid_from) + if value is not None + ] + known_values = [ + value for value in (existing["ingested_at"], ingested_at) + if value is not None + ] + desired_confidence = max( + float(existing["confidence"] or 0.0), + max(0.0, min(1.0, float(confidence))), + ) + desired_valid = min(valid_values) if valid_values else None + desired_known = min(known_values) if known_values else None + if ( + desired_confidence != float(existing["confidence"] or 0.0) + or desired_valid != existing["valid_from"] + or desired_known != existing["ingested_at"] + ): + self.conn.execute( + "UPDATE memory_entities SET confidence=?, valid_from=?, " + "ingested_at=? WHERE id=?", + ( + desired_confidence, desired_valid, desired_known, + existing["id"], + ), + ) + if commit: + self.conn.commit() + return existing["id"] + stamp = now_ts() + link_id = ids.new_id("edge") + self.conn.execute( + "INSERT INTO memory_entities(" + "id, memory_id, entity_id, workspace_id, repo_id, source_kind, confidence, " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, " + "provenance) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (link_id, memory_id, entity_id, workspace_id, repo_id, source_kind, + max(0.0, min(1.0, float(confidence))), + valid_from if valid_from is not None else stamp, + valid_to, valid_to_recorded_at, + ingested_at if ingested_at is not None else stamp, + expired_at, + _dumps(provenance or {})), + ) + if commit: + self.conn.commit() + return link_id + + def list_memory_entities(self, flt: Optional[SearchFilter] = None, *, + entity_ids: Optional[list[str]] = None, + limit: Optional[int] = None) -> list[dict]: + """Return bounded scoped/temporal incidence rows for graph retrieval.""" + valid_at, known_at = _temporal_anchors(flt) + sql = ( + "SELECT me.* FROM memory_entities me " + "JOIN memories m ON m.id=me.memory_id WHERE " + "(me.valid_from IS NULL OR me.valid_from<=?) " + "AND (me.valid_to IS NULL OR ? str: eid = edge.id or ids.new_id("edge") layer = normalize_graph_layer(edge.layer, edge.relation).value @@ -1579,7 +2242,7 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: incoming_provenance = _merge_edge_provenance([edge.provenance]) existing = self.conn.execute( "SELECT id, workspace_id, repo_id, src, dst, relation, layer, weight, " - "valid_from, valid_to, ingested_at, expired_at, provenance " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, provenance " "FROM edges WHERE id=?", (eid,) ).fetchone() replacing = existing is not None @@ -1613,17 +2276,29 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: value for value in (existing["valid_from"], edge.valid_from) if value is not None ) + desired_ingested_at = existing["ingested_at"] + if edge.ingested_at is not None: + desired_ingested_at = min( + value for value in (existing["ingested_at"], edge.ingested_at) + if value is not None + ) serialized_provenance = _dumps(merged_provenance) if desired_weight != float(existing["weight"] or 0.0) \ or desired_valid_from != existing["valid_from"] \ + or desired_ingested_at != existing["ingested_at"] \ or serialized_provenance != (existing["provenance"] or "{}"): self.conn.execute( - "UPDATE edges SET weight=?, valid_from=?, provenance=? WHERE id=?", - (desired_weight, desired_valid_from, serialized_provenance, eid), + "UPDATE edges SET weight=?, valid_from=?, ingested_at=?, " + "provenance=? WHERE id=?", + ( + desired_weight, desired_valid_from, desired_ingested_at, + serialized_provenance, eid, + ), ) self._write_edge_supports( eid, edge.relation, incoming_provenance, valid_from=edge.valid_from, valid_to=edge.valid_to, + valid_to_recorded_at=edge.valid_to_recorded_at, ingested_at=edge.ingested_at, expired_at=edge.expired_at, ) if commit: @@ -1632,7 +2307,7 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: equivalent = None if edge.valid_to is None and edge.expired_at is None: equivalent = self.conn.execute( - "SELECT id, weight, valid_from, provenance FROM edges " + "SELECT id, weight, valid_from, ingested_at, provenance FROM edges " "WHERE workspace_id IS ? AND repo_id IS ? AND src=? AND dst=? " "AND relation=? AND layer=? AND valid_to IS NULL AND expired_at IS NULL " "AND id<>? ORDER BY id LIMIT 1", @@ -1645,13 +2320,15 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: if replacing: closed_at = now_ts() self.conn.execute( - "UPDATE edges SET valid_to=? WHERE id=? AND valid_to IS NULL", - (closed_at, eid), + "UPDATE edges SET valid_to=?, valid_to_recorded_at=? " + "WHERE id=? AND valid_to IS NULL", + (closed_at, closed_at, eid), ) self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? " "AND valid_to IS NULL AND expired_at IS NULL", - (closed_at, eid), + (closed_at, closed_at, eid), ) existing_provenance = _loads(equivalent["provenance"], {}) merged_provenance = _merge_edge_provenance( @@ -1661,17 +2338,25 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: valid_values = [value for value in ( equivalent["valid_from"], edge.valid_from ) if value is not None] + known_values = [ + value for value in ( + equivalent["ingested_at"], edge.ingested_at + ) if value is not None + ] self.conn.execute( - "UPDATE edges SET weight=?, valid_from=?, provenance=? WHERE id=?", + "UPDATE edges SET weight=?, valid_from=?, ingested_at=?, provenance=? " + "WHERE id=?", ( max(float(equivalent["weight"] or 0.0), float(edge.weight or 0.0)), min(valid_values) if valid_values else now_ts(), + min(known_values) if known_values else now_ts(), _dumps(merged_provenance), equivalent["id"], ), ) self._write_edge_supports( equivalent["id"], edge.relation, incoming_provenance, valid_from=edge.valid_from, valid_to=edge.valid_to, + valid_to_recorded_at=edge.valid_to_recorded_at, ingested_at=edge.ingested_at, expired_at=edge.expired_at, ) if commit: @@ -1681,29 +2366,36 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: # ``upsert_edge`` replaces the supplied edge record. Close its previous # normalized evidence before writing the replacement so sources removed # from the new provenance cannot remain live invisibly. + closed_at = now_ts() self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? " "AND valid_to IS NULL AND expired_at IS NULL", - (now_ts(), eid), + (closed_at, closed_at, eid), ) self.conn.execute( "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " - "weight, valid_from, valid_to, ingested_at, expired_at, provenance) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) " + "weight, valid_from, valid_to, valid_to_recorded_at, ingested_at, " + "expired_at, provenance) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) " "ON CONFLICT(id) DO UPDATE SET workspace_id=excluded.workspace_id, " "repo_id=excluded.repo_id, src=excluded.src, dst=excluded.dst, " "relation=excluded.relation, layer=excluded.layer, weight=excluded.weight, " "valid_from=excluded.valid_from, valid_to=excluded.valid_to, " + "valid_to_recorded_at=excluded.valid_to_recorded_at, " "ingested_at=excluded.ingested_at, expired_at=excluded.expired_at, " "provenance=excluded.provenance", (eid, edge.workspace_id, edge.repo_id, source, target, edge.relation, layer, edge.weight, edge.valid_from if edge.valid_from is not None else now_ts(), - edge.valid_to, edge.ingested_at or now_ts(), edge.expired_at, + edge.valid_to, edge.valid_to_recorded_at, + edge.ingested_at if edge.ingested_at is not None else now_ts(), + edge.expired_at, _dumps(incoming_provenance)), ) self._write_edge_supports( eid, edge.relation, incoming_provenance, valid_from=edge.valid_from, valid_to=edge.valid_to, + valid_to_recorded_at=edge.valid_to_recorded_at, ingested_at=edge.ingested_at, expired_at=edge.expired_at, ) if commit: @@ -1711,18 +2403,24 @@ def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: return eid def invalidate_edge(self, edge_id: str, at: Optional[float] = None) -> None: - ts = now_ts() if at is None else at - self.conn.execute("UPDATE edges SET valid_to=? WHERE id=? AND valid_to IS NULL", - (ts, edge_id)) + recorded_at = now_ts() + ts = recorded_at if at is None else at self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " - "AND valid_to IS NULL AND expired_at IS NULL", (ts, edge_id) + "UPDATE edges SET valid_to=?, valid_to_recorded_at=? " + "WHERE id=? AND valid_to IS NULL", + (ts, recorded_at, edge_id), + ) + self.conn.execute( + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? AND valid_to IS NULL AND expired_at IS NULL", + (ts, recorded_at, edge_id), ) self.conn.commit() def _write_edge_supports(self, edge_id: str, relation: str, provenance: dict, *, valid_from: Optional[float] = None, valid_to: Optional[float] = None, + valid_to_recorded_at: Optional[float] = None, ingested_at: Optional[float] = None, expired_at: Optional[float] = None) -> None: source_kind = _edge_source_kind(provenance, relation) @@ -1772,13 +2470,17 @@ def _write_edge_supports(self, edge_id: str, relation: str, provenance: dict, self.conn.execute( "INSERT OR IGNORE INTO edge_supports " "(edge_id, memory_id, source_kind, confidence, valid_from, valid_to, " - "ingested_at, expired_at, provenance) VALUES (?,?,?,?,?,?,?,?,?)", + "valid_to_recorded_at, ingested_at, expired_at, provenance) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", (edge_id, memory_id, source_kind, confidence, - support_valid_from, valid_to, support_ingested_at, expired_at, + support_valid_from, valid_to, valid_to_recorded_at, + support_ingested_at, expired_at, _dumps(support_provenance)), ) def add_edge_support(self, edge_id: str, provenance: dict, *, + valid_from: Optional[float] = None, + ingested_at: Optional[float] = None, commit: bool = True) -> None: """Record another source memory supporting an existing graph edge.""" incoming = _provenance_memory_ids(provenance) @@ -1795,15 +2497,43 @@ def add_edge_support(self, edge_id: str, provenance: dict, *, self.conn.execute("UPDATE edges SET provenance=? WHERE id=?", (_dumps(merged_provenance), edge_id)) edge_row = self.conn.execute( - "SELECT relation, valid_from, valid_to, ingested_at, expired_at " + "SELECT relation, valid_from, valid_to, valid_to_recorded_at, " + "ingested_at, expired_at " "FROM edges WHERE id=?", (edge_id,) ).fetchone() if edge_row: + support_valid_from = ( + valid_from if valid_from is not None else edge_row["valid_from"] + ) + support_ingested_at = ( + ingested_at if ingested_at is not None else edge_row["ingested_at"] + ) self._write_edge_supports( edge_id, edge_row["relation"] or "", provenance, - valid_from=edge_row["valid_from"], valid_to=edge_row["valid_to"], - ingested_at=edge_row["ingested_at"], expired_at=edge_row["expired_at"], + valid_from=support_valid_from, valid_to=edge_row["valid_to"], + valid_to_recorded_at=edge_row["valid_to_recorded_at"], + ingested_at=support_ingested_at, expired_at=edge_row["expired_at"], ) + # The edge is the union of its supporting evidence intervals. A + # backdated support must make the relation visible at that earlier + # world time, and a historically imported support may likewise be + # known before the edge's previous system-time anchor. + valid_values = [ + value for value in (edge_row["valid_from"], support_valid_from) + if value is not None + ] + ingested_values = [ + value for value in (edge_row["ingested_at"], support_ingested_at) + if value is not None + ] + earlier_valid = min(valid_values) if valid_values else None + earlier_ingested = min(ingested_values) if ingested_values else None + if (earlier_valid != edge_row["valid_from"] + or earlier_ingested != edge_row["ingested_at"]): + self.conn.execute( + "UPDATE edges SET valid_from=?, ingested_at=? WHERE id=?", + (earlier_valid, earlier_ingested, edge_id), + ) if commit: self.conn.commit() @@ -1826,7 +2556,8 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N directly (service.py): those edges would carry provenance but no support rows, and would then silently never be invalidated. Normalize the edge writes first. """ - ts = at if at is not None else now_ts() + recorded_at = now_ts() + ts = at if at is not None else recorded_at owner = self.conn.fetchall( "SELECT workspace_id FROM memories WHERE id=?", (memory_id,)) workspace_id = owner[0]["workspace_id"] if owner else None @@ -1857,9 +2588,10 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N if memory_id not in supports: continue self.conn.execute( - "UPDATE edge_supports SET valid_to=? WHERE edge_id=? AND memory_id=? " + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + "WHERE edge_id=? AND memory_id=? " "AND valid_to IS NULL AND expired_at IS NULL", - (ts, row["id"], memory_id), + (ts, recorded_at, row["id"], memory_id), ) normalized_remaining = [r["memory_id"] for r in self.conn.execute( "SELECT DISTINCT memory_id FROM edge_supports WHERE edge_id=? " @@ -1876,12 +2608,16 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N (_dumps(prov), row["id"])) if ids_to_close: marks = ",".join("?" for _ in ids_to_close) - self.conn.execute(f"UPDATE edges SET valid_to=? WHERE id IN ({marks})", - (ts, *ids_to_close)) self.conn.execute( - f"UPDATE edge_supports SET valid_to=? WHERE edge_id IN ({marks}) " + f"UPDATE edges SET valid_to=?, valid_to_recorded_at=? " + f"WHERE id IN ({marks})", + (ts, recorded_at, *ids_to_close), + ) + self.conn.execute( + f"UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=? " + f"WHERE edge_id IN ({marks}) " "AND valid_to IS NULL AND expired_at IS NULL", - (ts, *ids_to_close), + (ts, recorded_at, *ids_to_close), ) if commit: self.conn.commit() @@ -1889,19 +2625,44 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N # ── memory-to-memory links (A-MEM style) ──────────────────────────────────── def edge_supports_in_scope(self, edge_ids: Optional[list[str]] = None, *, at: Optional[float] = None, + flt: Optional[SearchFilter] = None, limit: Optional[int] = None) -> list[dict]: - """Return live normalized evidence rows for graph inspection/scene scoring.""" - t = at if at is not None else now_ts() + """Return evidence visible at the supplied world/system-time anchors.""" + valid_at, known_at = _temporal_anchors(flt, valid_at=at) row_cap = None if limit is None else max(0, int(limit)) if row_cap == 0: return [] sql = ( - "SELECT id, edge_id, memory_id, source_kind, confidence, valid_from, " - "valid_to, ingested_at, expired_at, provenance FROM edge_supports " - "WHERE (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? None: + *, valid_from: Optional[float] = None, + valid_to: Optional[float] = None, + valid_to_recorded_at: Optional[float] = None, + ingested_at: Optional[float] = None, + expired_at: Optional[float] = None, + commit: bool = True) -> None: """Idempotent per (pair, relation): re-linking the same two memories with the same relation is a no-op in either direction, so auto-evolution and explicit ``engraphis_link`` calls can't accrete duplicate rows.""" - existing = self.conn.execute( - "SELECT rowid, layer, reason FROM mem_links " - "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? LIMIT 1", - (a, b, b, a, relation), - ).fetchone() - if existing: - updates: list[str] = [] - params: list[Any] = [] - if layer is not None: - graph_layer = normalize_graph_layer(layer, relation).value - if existing["layer"] != graph_layer: - updates.append("layer=?") - params.append(graph_layer) - if reason and existing["reason"] != reason: - updates.append("reason=?") - params.append(reason) - if updates: - params.append(existing["rowid"]) - self.conn.execute( - f"UPDATE mem_links SET {', '.join(updates)} WHERE rowid=?", - params, + requested_layer = ( + normalize_graph_layer(layer, relation).value + if layer is not None else None + ) + started_transaction = not self.conn.in_transaction + if started_transaction: + self.conn.execute("BEGIN IMMEDIATE") + try: + existing = self.conn.execute( + "SELECT rowid, a, b, relation, layer, reason, created_at, " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at " + "FROM mem_links " + "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? " + "AND valid_to IS NULL AND expired_at IS NULL " + "ORDER BY rowid DESC LIMIT 1", + (a, b, b, a, relation), + ).fetchone() + if existing: + graph_layer = ( + requested_layer + if requested_layer is not None else existing["layer"] ) - if commit: + replacement_reason = reason if reason else existing["reason"] + if ( + graph_layer != existing["layer"] + or replacement_reason != existing["reason"] + ): + # Metadata is part of what the system knew about this link. Updating + # it in place would rewrite a historical ``known_at`` view. Retire the + # system-time version and open a replacement over the same world-time + # interval so past reads remain immutable while current reads converge. + stamp = max( + now_ts(), + ( + float(existing["ingested_at"]) + if existing["ingested_at"] is not None + else float("-inf") + ), + ) + self.conn.execute( + "UPDATE mem_links SET expired_at=? " + "WHERE rowid=? AND expired_at IS NULL", + (stamp, existing["rowid"]), + ) + self.conn.execute( + "INSERT INTO mem_links(" + "a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,NULL)", + ( + existing["a"], existing["b"], existing["relation"], + graph_layer, replacement_reason, stamp, + existing["valid_from"], existing["valid_to"], + existing["valid_to_recorded_at"], stamp, + ), + ) + if commit: + self.conn.commit() + elif started_transaction: + # The pre-read reservation has no write to batch. Release it even + # for ``commit=False``; the old no-op path never opened a transaction. self.conn.commit() - return - graph_layer = normalize_graph_layer(layer, relation).value - self.conn.execute( - "INSERT INTO mem_links(a, b, relation, layer, reason, created_at) " - "VALUES (?,?,?,?,?,?)", - (a, b, relation, graph_layer, reason, now_ts()), - ) - if commit: - self.conn.commit() + return + graph_layer = requested_layer or normalize_graph_layer(None, relation).value + stamp = now_ts() + world_start = stamp if valid_from is None else valid_from + system_start = stamp if ingested_at is None else ingested_at + self.conn.execute( + "INSERT INTO mem_links(" + "a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (a, b, relation, graph_layer, reason, stamp, world_start, valid_to, + valid_to_recorded_at, system_start, expired_at), + ) + if commit: + self.conn.commit() + except BaseException: + if started_transaction and self.conn.in_transaction: + self.conn.rollback() + raise def has_link(self, a: str, b: str, *, relation: Optional[str] = None) -> bool: - sql = "SELECT 1 FROM mem_links WHERE ((a=? AND b=?) OR (a=? AND b=?))" + """Return whether the pair has a current open link interval. + + Closed history must not block a later reactivation of the same relationship. + Historical visibility remains available through ``get_links``/``links_among``. + """ + sql = ( + "SELECT 1 FROM mem_links WHERE ((a=? AND b=?) OR (a=? AND b=?)) " + "AND valid_to IS NULL AND expired_at IS NULL" + ) params: list[Any] = [a, b, b, a] if relation is not None: sql += " AND relation=?" params.append(relation) return self.conn.execute(sql + " LIMIT 1", params).fetchone() is not None - def get_links(self, memory_id: str) -> list[dict]: + def get_links(self, memory_id: str, *, + flt: Optional[SearchFilter] = None) -> list[dict]: + """Return direct links visible at the filter's bi-temporal anchors.""" + visible_sql, params = _temporal_visibility_sql("", flt) rows = self.conn.execute( - "SELECT a, b, relation, layer, reason, created_at " - "FROM mem_links WHERE a=? OR b=?", - (memory_id, memory_id), + "SELECT a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at FROM mem_links " + f"WHERE (a=? OR b=?) AND {visible_sql} ORDER BY a, b, relation", + (memory_id, memory_id, *params), ).fetchall() return [dict(r) for r in rows] def edges_in_scope(self, flt: Optional[SearchFilter] = None, *, at: Optional[float] = None, limit: Optional[int] = None) -> list[Edge]: - """Every edge valid at ``at`` within the filter's workspace/repo — the graph - the PPR retrieval arm walks (edges outside their validity window are invisible, - same bi-temporal rule as memories).""" - t = at if at is not None else now_ts() + """Edges visible at ``at``/``filter.valid_at`` and ``filter.known_at``.""" + valid_at, known_at = _temporal_anchors(flt, valid_at=at) sql = ("SELECT * FROM edges WHERE (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? list[dict]: - """mem_links rows where *both* endpoints are in ``ids`` (for graph retrieval).""" + layers: Optional[list[GraphLayer]] = None, + flt: Optional[SearchFilter] = None, + limit: Optional[int] = None) -> list[dict]: + """Return memory links visible under both temporal anchors. + + Chunk only the indexed ``a`` side and filter ``b`` against an in-memory set. + This keeps every statement below SQLite's portable variable limit while + preserving exact pair semantics for graphs containing thousands of memories. + """ if not ids: return [] - marks = ",".join("?" for _ in ids) - sql = ( - f"SELECT a, b, relation, layer, reason FROM mem_links " - f"WHERE a IN ({marks}) AND b IN ({marks})" - ) - params: list[Any] = [*ids, *ids] - if layers: - layer_marks = ",".join("?" for _ in layers) - sql += f" AND layer IN ({layer_marks})" - params.extend(_enum(layer) for layer in layers) - rows = self.conn.execute(sql, params).fetchall() - return [dict(r) for r in rows] + if layers is not None and not layers: + return [] + row_cap = None if limit is None else max(0, int(limit)) + if row_cap == 0: + return [] + wanted = set(ids) + ordered_ids = sorted(wanted) + visibility_sql, visibility_params = _temporal_visibility_sql("", flt) + rows: list[dict] = [] + # Leave headroom for the time anchor and optional layer parameters. + chunk_size = max(1, IN_CLAUSE_CHUNK - 16) + for start in range(0, len(ordered_ids), chunk_size): + if row_cap is not None and len(rows) >= row_cap: + break + chunk = ordered_ids[start:start + chunk_size] + marks = ",".join("?" for _ in chunk) + sql = ( + "SELECT a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at FROM mem_links " + f"WHERE a IN ({marks}) " + f"AND {visibility_sql}" + ) + params: list[Any] = [*chunk, *visibility_params] + if layers is not None: + layer_marks = ",".join("?" for _ in layers) + sql += f" AND layer IN ({layer_marks})" + params.extend(_enum(layer) for layer in layers) + sql += " ORDER BY a, b, relation" + found = self.conn.execute(sql, params).fetchall() + for row in found: + if row["b"] not in wanted: + continue + rows.append(dict(row)) + if row_cap is not None and len(rows) >= row_cap: + break + return rows def neighbors(self, node_ids: list[str], *, at: Optional[float] = None, - layers: Optional[list[GraphLayer]] = None) -> list[Edge]: + layers: Optional[list[GraphLayer]] = None, + flt: Optional[SearchFilter] = None) -> list[Edge]: if not node_ids: return [] - t = at if at is not None else now_ts() + valid_at, known_at = _temporal_anchors(flt, valid_at=at) marks = ",".join("?" for _ in node_ids) sql = ( f"SELECT * FROM edges WHERE (src IN ({marks}) OR dst IN ({marks})) " - f"AND (valid_from IS NULL OR valid_from<=?) AND (valid_to IS NULL OR ? None: - """Re-indexing a file replaces its symbols/edges — incremental indexing is - idempotent per file, not additive.""" + """Retire a file's live code graph rows before an incremental re-index.""" + stamp = now_ts() symbol_rows = self.conn.execute( - "SELECT id FROM symbols WHERE repo_id=? AND file=?", (repo_id, file) + "SELECT id FROM symbols WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", (repo_id, file) ).fetchall() symbol_ids = [row["id"] for row in symbol_rows] if symbol_ids: marks = ",".join("?" for _ in symbol_ids) self.conn.execute( - f"DELETE FROM code_memory_links WHERE repo_id=? " - f"AND symbol_id IN ({marks})", - (repo_id, *symbol_ids), + f"UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + f"WHERE repo_id=? " + f"AND symbol_id IN ({marks}) AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, *symbol_ids), ) - self.conn.execute("DELETE FROM symbols WHERE repo_id=? AND file=?", (repo_id, file)) - self.conn.execute("DELETE FROM code_edges WHERE repo_id=? AND file=?", (repo_id, file)) + self.conn.execute( + "UPDATE symbols SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, file), + ) + self.conn.execute( + "UPDATE code_edges SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? AND file=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, file), + ) if commit: self.conn.commit() @@ -2079,10 +2975,10 @@ def upsert_symbol(self, *, repo_id: str, kind: str, name: str, fqname: str, file sid = ids.new_id("symbol") self.conn.execute( "INSERT INTO symbols(id, repo_id, kind, name, fqname, file, span, signature, " - "docstring, lang, exported, content_hash, updated_at) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "docstring, lang, exported, content_hash, updated_at, valid_from, ingested_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (sid, repo_id, kind, name, fqname, file, span, signature, docstring, - lang, int(exported), content_hash, now_ts()), + lang, int(exported), content_hash, now_ts(), now_ts(), now_ts()), ) if commit: self.conn.commit() @@ -2096,9 +2992,10 @@ def add_code_edge(self, *, repo_id: str, src: str, dst: str, relation: str, if layer is None and graph_layer == GraphLayer.SEMANTIC: graph_layer = GraphLayer.ENTITY self.conn.execute( - "INSERT INTO code_edges(id, repo_id, src, dst, relation, layer, file, line) " - "VALUES (?,?,?,?,?,?,?,?)", - (eid, repo_id, src, dst, relation, graph_layer.value, file, line), + "INSERT INTO code_edges(id, repo_id, src, dst, relation, layer, file, line, " + "valid_from, ingested_at) VALUES (?,?,?,?,?,?,?,?,?,?)", + (eid, repo_id, src, dst, relation, graph_layer.value, file, line, + now_ts(), now_ts()), ) if commit: self.conn.commit() @@ -2159,9 +3056,11 @@ def update_repo_index(self, repo_id: str, *, root_path: str, ) self.conn.commit() - def list_symbols(self, repo_id: str, *, limit: Optional[int] = None) -> list[dict]: - sql = "SELECT * FROM symbols WHERE repo_id=? ORDER BY file, fqname" - params: list[Any] = [repo_id] + def list_symbols(self, repo_id: str, *, limit: Optional[int] = None, + flt: Optional[SearchFilter] = None) -> list[dict]: + temporal, params = _temporal_visibility_sql("", flt) + sql = "SELECT * FROM symbols WHERE repo_id=? AND " + temporal + " ORDER BY file, fqname" + params = [repo_id, *params] if limit is not None: sql += " LIMIT ?" params.append(max(0, int(limit))) # never -1 == SQLite "unlimited" @@ -2169,9 +3068,11 @@ def list_symbols(self, repo_id: str, *, limit: Optional[int] = None) -> list[dic def list_symbols_page(self, repo_id: str, *, after: Optional[tuple[str, str, str]] = None, - limit: int = 500) -> list[dict]: - sql = "SELECT * FROM symbols WHERE repo_id=?" - params: list[Any] = [repo_id] + limit: int = 500, + flt: Optional[SearchFilter] = None) -> list[dict]: + temporal, params = _temporal_visibility_sql("", flt) + sql = "SELECT * FROM symbols WHERE repo_id=? AND " + temporal + params = [repo_id, *params] if after is not None: file, fqname, symbol_id = after sql += ( @@ -2184,9 +3085,11 @@ def list_symbols_page(self, repo_id: str, *, return [dict(row) for row in self.conn.execute(sql, params).fetchall()] def list_code_edges(self, repo_id: str, *, limit: Optional[int] = None, - layers: Optional[list[GraphLayer]] = None) -> list[dict]: - sql = "SELECT * FROM code_edges WHERE repo_id=?" - params: list[Any] = [repo_id] + layers: Optional[list[GraphLayer]] = None, + flt: Optional[SearchFilter] = None) -> list[dict]: + temporal, params = _temporal_visibility_sql("", flt) + sql = "SELECT * FROM code_edges WHERE repo_id=? AND " + temporal + params = [repo_id, *params] if layers is not None: if not layers: return [] @@ -2199,68 +3102,88 @@ def list_code_edges(self, repo_id: str, *, limit: Optional[int] = None, params.append(max(0, int(limit))) # never -1 == SQLite "unlimited" return [dict(r) for r in self.conn.execute(sql, params).fetchall()] - def symbols_for_files(self, repo_id: str, files: list[str]) -> list[dict]: + def symbols_for_files(self, repo_id: str, files: list[str], *, + flt: Optional[SearchFilter] = None) -> list[dict]: if not files: return [] marks = ",".join("?" for _ in files) + temporal, params = _temporal_visibility_sql("", flt) rows = self.conn.execute( f"SELECT * FROM symbols WHERE repo_id=? AND file IN ({marks}) " - "ORDER BY file, fqname", - (repo_id, *files), + f"AND {temporal} ORDER BY file, fqname", + (repo_id, *files, *params), ).fetchall() return [dict(r) for r in rows] def count_code_edges(self, repo_id: str) -> int: row = self.conn.execute( - "SELECT COUNT(*) AS n FROM code_edges WHERE repo_id=?", (repo_id,) + "SELECT COUNT(*) AS n FROM code_edges WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", (repo_id,) ).fetchone() return int(row["n"]) if row else 0 - def search_symbols(self, repo_id: str, query: str, *, limit: int = 20) -> list[dict]: + def search_symbols(self, repo_id: str, query: str, *, limit: int = 20, + flt: Optional[SearchFilter] = None) -> list[dict]: """Substring match on name/fqname (no embedding yet — v1 is lexical).""" like = f"%{_escape_like(query)}%" + temporal, temporal_params = _temporal_visibility_sql("", flt) rows = self.conn.execute( - "SELECT * FROM symbols WHERE repo_id=? AND (name LIKE ? ESCAPE '\\' OR fqname LIKE ? ESCAPE '\\') " + f"SELECT * FROM symbols WHERE repo_id=? AND {temporal} " + "AND (name LIKE ? ESCAPE '\\' OR fqname LIKE ? ESCAPE '\\') " "ORDER BY name LIMIT ?", - (repo_id, like, like, limit), + (repo_id, *temporal_params, like, like, limit), ).fetchall() return [dict(r) for r in rows] - def get_symbol_callers(self, repo_id: str, name: str, *, limit: int = 50) -> list[dict]: + def get_symbol_callers(self, repo_id: str, name: str, *, limit: int = 50, + flt: Optional[SearchFilter] = None) -> list[dict]: + temporal, temporal_params = _temporal_visibility_sql("", flt) rows = self.conn.execute( - "SELECT * FROM code_edges WHERE repo_id=? AND dst=? AND relation='calls' LIMIT ?", - (repo_id, name, limit), + "SELECT * FROM code_edges WHERE repo_id=? AND dst=? AND relation='calls' " + f"AND {temporal} LIMIT ?", + (repo_id, name, *temporal_params, limit), ).fetchall() return [dict(r) for r in rows] def count_symbols(self, repo_id: str) -> int: row = self.conn.execute( - "SELECT COUNT(*) AS n FROM symbols WHERE repo_id=?", (repo_id,) + "SELECT COUNT(*) AS n FROM symbols WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", (repo_id,) ).fetchone() return int(row["n"]) if row else 0 def link_memory_symbol(self, *, repo_id: str, symbol_id: str, memory_id: str, relation: str = "mentions", confidence: float = 1.0, commit: bool = True) -> str: + existing = self.conn.execute( + "SELECT id FROM code_memory_links WHERE repo_id=? AND symbol_id=? " + "AND memory_id=? AND relation=? AND valid_to IS NULL AND expired_at IS NULL", + (repo_id, symbol_id, memory_id, relation), + ).fetchone() + if existing is not None: + return existing["id"] link_id = ids.new_id("edge") + stamp = now_ts() self.conn.execute( "INSERT OR IGNORE INTO code_memory_links(" - "id, repo_id, symbol_id, memory_id, relation, confidence, created_at" - ") VALUES (?,?,?,?,?,?,?)", + "id, repo_id, symbol_id, memory_id, relation, confidence, created_at, " + "valid_from, ingested_at" + ") VALUES (?,?,?,?,?,?,?,?,?)", (link_id, repo_id, symbol_id, memory_id, relation, - max(0.0, min(1.0, float(confidence))), now_ts()), + max(0.0, min(1.0, float(confidence))), stamp, stamp, stamp), ) - row = self.conn.execute( - "SELECT id FROM code_memory_links WHERE repo_id=? AND symbol_id=? " - "AND memory_id=? AND relation=?", - (repo_id, symbol_id, memory_id, relation), - ).fetchone() if commit: self.conn.commit() - return row["id"] if row else link_id + return link_id def clear_code_memory_links(self, repo_id: str, *, commit: bool = True) -> None: - self.conn.execute("DELETE FROM code_memory_links WHERE repo_id=?", (repo_id,)) + stamp = now_ts() + self.conn.execute( + "UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id), + ) if commit: self.conn.commit() @@ -2269,9 +3192,12 @@ def clear_code_memory_links_for_memories(self, repo_id: str, memory_ids: list[st if not memory_ids: return marks = ",".join("?" for _ in memory_ids) + stamp = now_ts() self.conn.execute( - f"DELETE FROM code_memory_links WHERE repo_id=? AND memory_id IN ({marks})", - (repo_id, *memory_ids), + f"UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + f"WHERE repo_id=? " + f"AND memory_id IN ({marks}) AND valid_to IS NULL AND expired_at IS NULL", + (stamp, stamp, repo_id, *memory_ids), ) if commit: self.conn.commit() @@ -2280,12 +3206,14 @@ def prune_code_memory_links(self, repo_id: str, *, commit: bool = True) -> None: """Remove bridges whose repo-associated memory is no longer live.""" t = now_ts() self.conn.execute( - "DELETE FROM code_memory_links WHERE repo_id=? AND NOT EXISTS (" + "UPDATE code_memory_links SET valid_to=?, valid_to_recorded_at=? " + "WHERE repo_id=? " + "AND valid_to IS NULL AND expired_at IS NULL AND NOT EXISTS (" "SELECT 1 FROM memories AS m WHERE m.id=code_memory_links.memory_id AND m.repo_id=? " "AND (m.valid_from IS NULL OR m.valid_from<=?) " "AND (m.valid_to IS NULL OR ? list[dict]: sql = ( "SELECT l.*, s.name, s.fqname, s.file, s.kind AS symbol_kind, " - "m.title, m.mtype, m.valid_to, m.expired_at " + "m.title, m.mtype, m.valid_to AS memory_valid_to, " + "m.expired_at AS memory_expired_at " "FROM code_memory_links l " - "LEFT JOIN symbols s ON s.id=l.symbol_id " - "LEFT JOIN memories m ON m.id=l.memory_id " + "JOIN symbols s ON s.id=l.symbol_id " + "JOIN memories m ON m.id=l.memory_id " "WHERE l.repo_id=?" ) params: list[Any] = [repo_id] - if flt is not None: - where, visibility_params = self._where(flt, include_invalid=False, alias="m") - if where: - sql += " AND " + " AND ".join(where) - params.extend(visibility_params) + link_visibility, link_params = _temporal_visibility_sql("l", flt) + sql += " AND " + link_visibility + params.extend(link_params) + symbol_visibility, symbol_params = _temporal_visibility_sql("s", flt) + sql += " AND " + symbol_visibility + params.extend(symbol_params) + where, visibility_params = self._where(flt, include_invalid=False, alias="m") + if where: + sql += " AND " + " AND ".join(where) + params.extend(visibility_params) sql += " ORDER BY l.created_at, l.id" if limit is not None: sql += " LIMIT ?" @@ -2325,6 +3259,9 @@ def memories_for_symbol(self, repo_id: str, symbol_id: str, *, "WHERE l.repo_id=? AND l.symbol_id=?" ) params: list[Any] = [repo_id, symbol_id] + link_visibility, link_params = _temporal_visibility_sql("l", flt) + sql += " AND " + link_visibility + params.extend(link_params) where, visibility_params = self._where(flt, include_invalid=False, alias="m") if where: sql += " AND " + " AND ".join(where) @@ -2341,12 +3278,60 @@ def memories_for_symbol(self, repo_id: str, symbol_id: str, *, out.append(item) return out - def symbols_for_memory(self, repo_id: str, memory_id: str) -> list[dict]: + def memories_for_symbols(self, repo_id: str, symbol_ids: list[str], *, + flt: Optional[SearchFilter] = None, + limit: int = 20) -> dict[str, list[dict]]: + """Return a bounded memory ranking for many symbols in one SQL query.""" + unique_ids = list(dict.fromkeys( + str(symbol_id) for symbol_id in symbol_ids if str(symbol_id) + ))[:500] + if not unique_ids: + return {} + per_symbol_limit = max(1, min(100, int(limit))) + placeholders = ",".join("?" for _ in unique_ids) + sql = ( + "WITH ranked AS (" + "SELECT l.symbol_id, m.id, m.title, m.content, m.mtype, m.scope, " + "m.importance, m.provenance, l.relation, l.confidence, " + "ROW_NUMBER() OVER (PARTITION BY l.symbol_id " + "ORDER BY l.confidence DESC, m.importance DESC, " + "m.ingested_at DESC, l.id, m.id) AS row_rank " + "FROM code_memory_links l JOIN memories m ON m.id=l.memory_id " + f"WHERE l.repo_id=? AND l.symbol_id IN ({placeholders})" + ) + params: list[Any] = [repo_id, *unique_ids] + link_visibility, link_params = _temporal_visibility_sql("l", flt) + sql += " AND " + link_visibility + params.extend(link_params) + where, visibility_params = self._where(flt, include_invalid=False, alias="m") + if where: + sql += " AND " + " AND ".join(where) + params.extend(visibility_params) + sql += ( + ") SELECT symbol_id, id, title, content, mtype, scope, importance, " + "provenance, relation, confidence FROM ranked WHERE row_rank<=? " + "ORDER BY symbol_id, row_rank" + ) + params.append(per_symbol_limit) + grouped: dict[str, list[dict]] = {} + for row in self.conn.execute(sql, params).fetchall(): + item = dict(row) + symbol_id = str(item.pop("symbol_id")) + item["provenance"] = _loads(item.get("provenance"), {}) + grouped.setdefault(symbol_id, []).append(item) + return grouped + + def symbols_for_memory(self, repo_id: str, memory_id: str, *, + flt: Optional[SearchFilter] = None) -> list[dict]: + link_visibility, link_params = _temporal_visibility_sql("l", flt) + symbol_visibility, symbol_params = _temporal_visibility_sql("s", flt) rows = self.conn.execute( "SELECT s.*, l.relation, l.confidence FROM code_memory_links l " "JOIN symbols s ON s.id=l.symbol_id " - "WHERE l.repo_id=? AND l.memory_id=? ORDER BY l.confidence DESC, s.fqname", - (repo_id, memory_id), + f"WHERE l.repo_id=? AND l.memory_id=? AND {link_visibility} " + f"AND {symbol_visibility} " + "ORDER BY l.confidence DESC, s.fqname", + (repo_id, memory_id, *link_params, *symbol_params), ).fetchall() return [dict(row) for row in rows] @@ -2403,6 +3388,149 @@ def audit(self, actor: str, action: str, target: str, detail: str = "", if commit: self.conn.commit() + def _backfill_receipt_sequences(self) -> None: + """Assign durable logical ordinals once when the sequence column is introduced.""" + scopes = self.conn.execute( + "SELECT DISTINCT workspace_id FROM operation_receipts" + ).fetchall() + for scope in scopes: + workspace_id = str(scope["workspace_id"] or "") + chain = self._receipt_chain_state(workspace_id) + for sequence, row in enumerate(chain["rows"], 1): + self.conn.execute( + "UPDATE operation_receipts SET sequence=? WHERE id=?", + (sequence, row["id"]), + ) + + def _receipt_chain_state(self, workspace_id: str) -> dict: + """Reconstruct one receipt chain from immutable predecessor hashes. + + SQLite ``rowid`` is physical placement, not durable ordering: VACUUM and table + rewrites may renumber it. The receipt payload already carries the true linked-list + order, while ``receipt_chain_heads`` anchors the expected tail. This helper keeps + traversal independent of storage layout and returns a deterministic fallback order + when corruption makes a single chain impossible. + """ + rows = [dict(row) for row in self.conn.execute( + "SELECT id, sequence, payload, prev_hash, receipt_hash " + "FROM operation_receipts " + "WHERE workspace_id=?", + (workspace_id,), + ).fetchall()] + + def text(value: Any) -> str: + return value if isinstance(value, str) else str(value or "") + + def stable_key(row: dict) -> tuple[str, str]: + material = "\0".join(( + text(row.get("receipt_hash")), + text(row.get("prev_hash")), + text(row.get("id")), + hashlib.sha256(text(row.get("payload")).encode("utf-8")).hexdigest(), + )) + return hashlib.sha256(material.encode("utf-8")).hexdigest(), material + + children: dict[str, list[dict]] = {} + for row in rows: + children.setdefault(text(row.get("prev_hash")), []).append(row) + for candidates in children.values(): + candidates.sort(key=stable_key) + + structure_errors: list[dict] = [] + ordered: list[dict] = [] + roots = children.get("", []) + if rows and len(roots) != 1: + structure_errors.append({ + "index": 0, + "id": "", + "error": "chain_root_count", + }) + if len(roots) == 1: + current = roots[0] + visited_hashes: set[str] = set() + while current is not None: + receipt_hash = text(current.get("receipt_hash")) + if receipt_hash in visited_hashes: + structure_errors.append({ + "index": len(ordered), + "id": text(current.get("id")), + "error": "chain_cycle", + }) + break + visited_hashes.add(receipt_hash) + ordered.append(current) + successors = children.get(receipt_hash, []) + if len(successors) > 1: + structure_errors.append({ + "index": len(ordered) - 1, + "id": text(current.get("id")), + "error": "chain_fork", + }) + break + current = successors[0] if successors else None + + ordered_identity = {id(row) for row in ordered} + if len(ordered) != len(rows): + structure_errors.append({ + "index": len(ordered), + "id": "", + "error": "chain_disconnected", + }) + ordered.extend(sorted( + (row for row in rows if id(row) not in ordered_identity), + key=stable_key, + )) + + row_errors: list[dict] = [] + for index, row in enumerate(ordered): + if type(row.get("sequence")) is not int or row["sequence"] != index + 1: + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "sequence_mismatch", + }) + raw = text(row.get("payload")) + stored_hash = text(row.get("receipt_hash")) + if hashlib.sha256(raw.encode("utf-8")).hexdigest() != stored_hash: + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "hash_mismatch", + }) + try: + payload = json.loads(raw) + except (TypeError, ValueError, RecursionError): + payload = None + if ( + not isinstance(payload, dict) + or payload.get("id") != row.get("id") + or payload.get("prev_hash") != row.get("prev_hash") + ): + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "payload_mismatch", + }) + if _public_receipt_row(row).get("invalid_payload") is True: + row_errors.append({ + "index": index, + "id": text(row.get("id")), + "error": "payload_schema_invalid", + }) + + structurally_valid = not structure_errors and len(ordered) == len(rows) + head = ( + text(ordered[-1].get("receipt_hash")) + if ordered and structurally_valid else "" + ) + return { + "rows": ordered, + "head": head, + "structure_errors": structure_errors, + "row_errors": row_errors, + "errors": [*row_errors, *structure_errors], + } + def record_receipt(self, operation: str, *, workspace_id: str = "", repo_id: str = "", actor: str = "system", target_count: int = 0, status: str = "ok", @@ -2415,7 +3543,24 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", is anchored independently, so modification, reordering, interior deletion, and tail truncation are detectable during verification. """ - operation = str(operation or "unknown")[:80] + operation = str(operation or "unknown") + operation_normalized = operation.strip().casefold() + operation = ( + operation_normalized + if operation_normalized in _PUBLIC_RECEIPT_OPERATIONS + else "sha256:" + hashlib.sha256(operation.encode("utf-8")).hexdigest() + ) + raw_status = str(status or "ok") + status_normalized = raw_status.strip().casefold() + safe_status = ( + status_normalized + if status_normalized in _PUBLIC_RECEIPT_STATUSES + else "sha256:" + hashlib.sha256(raw_status.encode("utf-8")).hexdigest() + ) + try: + safe_target_count = max(0, int(target_count)) + except (TypeError, ValueError, OverflowError): + safe_target_count = 0 actor = str(actor or "system")[:200] workspace_id = str(workspace_id or "") repo_id = str(repo_id or "") @@ -2433,15 +3578,6 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", f"{workspace_id}\0{repo_id}".encode("utf-8") ).hexdigest()[:24] actor_digest = hashlib.sha256(actor.encode("utf-8")).hexdigest()[:16] - chain = self.conn.execute( - "SELECT COUNT(*) AS n, " - "COALESCE((SELECT receipt_hash FROM operation_receipts " - "WHERE workspace_id=? ORDER BY rowid DESC LIMIT 1), '') AS head " - "FROM operation_receipts WHERE workspace_id=?", - (workspace_id, workspace_id), - ).fetchone() - current_count = int(chain["n"] or 0) - prev_hash = str(chain["head"] or "") anchor = self.conn.execute( "SELECT receipt_count, head_hash, integrity_error " "FROM receipt_chain_heads " @@ -2449,15 +3585,72 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", (workspace_id,), ).fetchone() anchor_error = str(anchor["integrity_error"] or "") if anchor else "" - if anchor is not None and ( - int(anchor["receipt_count"]) != current_count - or str(anchor["head_hash"]) != prev_hash - ): - # Preserve evidence of the mismatch without bricking the operation that - # requested this receipt. The new receipt continues from the rows that - # actually remain, while verification stays invalid until an explicit - # repair/export decision clears the persistent integrity marker. - anchor_error = anchor_error or "pre_append_anchor_mismatch" + latest = self.conn.execute( + "SELECT sequence FROM operation_receipts " + "WHERE workspace_id=? ORDER BY sequence DESC LIMIT 1", + (workspace_id,), + ).fetchone() + current_count: Optional[int] = None + prev_hash = "" + if anchor is None and latest is None: + # First receipt for a workspace: no scan and no anchor are expected. + current_count = 0 + elif anchor is not None: + anchor_count = anchor["receipt_count"] + anchor_head = anchor["head_hash"] + if ( + type(anchor_count) is int + and anchor_count == 0 + and anchor_head == "" + and latest is None + ): + current_count = 0 + elif ( + type(anchor_count) is int + and anchor_count > 0 + and latest is not None + and latest["sequence"] == anchor_count + ): + head_row = self.conn.execute( + "SELECT id, sequence, payload, prev_hash, receipt_hash " + "FROM operation_receipts " + "WHERE workspace_id=? AND sequence=?", + (workspace_id, anchor_count), + ).fetchone() + if ( + head_row is not None + and head_row["receipt_hash"] == anchor_head + and not _public_receipt_row(dict(head_row)).get( + "invalid_payload", False + ) + ): + current_count = anchor_count + prev_hash = str(anchor_head) + + if current_count is None: + # The independently stored anchor/ordinal did not describe a healthy + # head. Reconstruct only on this exceptional path so a safe unique + # predecessor can still be extended without retrying the memory action. + chain = self._receipt_chain_state(workspace_id) + if chain["structure_errors"]: + raise sqlite3.IntegrityError( + "receipt chain has no unique structural head; append refused" + ) + current_count = len(chain["rows"]) + prev_hash = str(chain["head"] or "") + if chain["row_errors"]: + anchor_error = anchor_error or "pre_append_chain_corruption" + if anchor is None and current_count: + anchor_error = anchor_error or "pre_append_anchor_missing" + elif anchor is not None and ( + type(anchor["receipt_count"]) is not int + or anchor["receipt_count"] != current_count + or str(anchor["head_hash"]) != prev_hash + ): + # Keep evidence of deletion or anchor damage while extending the + # unique chain that actually remains. + anchor_error = anchor_error or "pre_append_anchor_mismatch" + next_sequence = current_count + 1 safe_meta = _receipt_metadata(metadata or {}) payload_obj = { "version": 1, @@ -2466,8 +3659,8 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", "operation": operation, "scope_digest": scope_digest, "actor_digest": actor_digest, - "target_count": max(0, int(target_count)), - "status": str(status or "ok")[:40], + "target_count": safe_target_count, + "status": safe_status, "metadata": safe_meta, "prev_hash": prev_hash, } @@ -2477,10 +3670,11 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", receipt_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() self.conn.execute( "INSERT INTO operation_receipts(id, ts, operation, workspace_id, repo_id, " - "scope_digest, actor, target_count, status, payload, prev_hash, " - "receipt_hash) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + "sequence, scope_digest, actor, target_count, status, payload, prev_hash, " + "receipt_hash) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( - receipt_id, ts, operation, workspace_id, repo_id, scope_digest, + receipt_id, ts, operation, workspace_id, repo_id, next_sequence, + scope_digest, actor_digest, payload_obj["target_count"], payload_obj["status"], payload, prev_hash, receipt_hash, ), @@ -2507,40 +3701,21 @@ def record_receipt(self, operation: str, *, workspace_id: str = "", raise def list_receipts(self, *, workspace_id: str, limit: int = 100) -> list[dict]: + safe_limit = max(1, min(10_000, int(limit))) rows = self.conn.execute( - "SELECT payload, receipt_hash FROM operation_receipts WHERE workspace_id=? " - "ORDER BY rowid DESC LIMIT ?", - (workspace_id, max(1, min(10_000, int(limit)))), + "SELECT id, sequence, payload, prev_hash, receipt_hash " + "FROM operation_receipts WHERE workspace_id=? " + "ORDER BY sequence DESC LIMIT ?", + (workspace_id, safe_limit), ).fetchall() - out = [] - for row in rows: - payload = _loads(row["payload"], {}) - if isinstance(payload, dict): - payload["hash"] = row["receipt_hash"] - out.append(payload) - return out + return [_public_receipt_row(dict(row)) for row in rows] def verify_receipts(self, *, workspace_id: str, expected_head: str = "", expected_count: Optional[int] = None) -> dict: - rows = self.conn.execute( - "SELECT id, payload, prev_hash, receipt_hash FROM operation_receipts " - "WHERE workspace_id=? ORDER BY rowid ASC", - (workspace_id,), - ).fetchall() - previous = "" - errors: list[dict] = [] - for index, row in enumerate(rows): - actual = hashlib.sha256(row["payload"].encode("utf-8")).hexdigest() - if actual != row["receipt_hash"]: - errors.append({"index": index, "id": row["id"], "error": "hash_mismatch"}) - payload = _loads(row["payload"], {}) - if not isinstance(payload, dict) or payload.get("id") != row["id"] \ - or payload.get("prev_hash") != row["prev_hash"]: - errors.append({"index": index, "id": row["id"], - "error": "payload_mismatch"}) - if row["prev_hash"] != previous: - errors.append({"index": index, "id": row["id"], "error": "chain_break"}) - previous = row["receipt_hash"] + chain = self._receipt_chain_state(workspace_id) + rows = chain["rows"] + errors: list[dict] = list(chain["errors"]) + head = str(chain["head"] or "") anchor = self.conn.execute( "SELECT receipt_count, head_hash, integrity_error " "FROM receipt_chain_heads WHERE workspace_id=?", @@ -2549,11 +3724,12 @@ def verify_receipts(self, *, workspace_id: str, expected_head: str = "", if rows and anchor is None: errors.append({"index": len(rows), "id": "", "error": "missing_anchor"}) elif anchor is not None: - if int(anchor["receipt_count"]) != len(rows): + anchor_count = anchor["receipt_count"] + if type(anchor_count) is not int or anchor_count < 0 or anchor_count != len(rows): errors.append({ "index": len(rows), "id": "", "error": "anchor_count_mismatch", }) - if str(anchor["head_hash"]) != previous: + if str(anchor["head_hash"]) != head: errors.append({ "index": len(rows), "id": "", "error": "anchor_head_mismatch", }) @@ -2562,7 +3738,7 @@ def verify_receipts(self, *, workspace_id: str, expected_head: str = "", "index": len(rows), "id": "", "error": "anchor_integrity_error", }) expected_head = str(expected_head or "").strip() - if expected_head and previous != expected_head: + if expected_head and head != expected_head: errors.append({ "index": len(rows), "id": "", "error": "expected_head_mismatch", }) @@ -2578,7 +3754,7 @@ def verify_receipts(self, *, workspace_id: str, expected_head: str = "", return { "valid": not errors, "count": len(rows), - "head": previous, + "head": head, "anchored": anchor is not None, "errors": errors, } @@ -2656,12 +3832,19 @@ def _where(self, flt: Optional[SearchFilter], include_invalid: bool, where.append(f"{p}mtype IN ({marks})") params.extend(_enum(m) for m in flt.mtypes) if not include_invalid: - t = (flt.as_of if flt and flt.as_of is not None else now_ts()) + valid_at, known_at = _temporal_anchors(flt) where.append(f"({p}valid_from IS NULL OR {p}valid_from<=?)") - params.append(t) - where.append(f"({p}valid_to IS NULL OR ?<{p}valid_to)") - params.append(t) - where.append(f"{p}expired_at IS NULL") + params.append(valid_at) + where.append( + f"({p}valid_to IS NULL OR ?<{p}valid_to OR " + f"({p}valid_to_recorded_at IS NOT NULL " + f"AND ?<{p}valid_to_recorded_at))" + ) + params.extend((valid_at, known_at)) + where.append(f"({p}ingested_at IS NULL OR {p}ingested_at<=?)") + params.append(known_at) + where.append(f"({p}expired_at IS NULL OR ?<{p}expired_at)") + params.append(known_at) return where, params @@ -2681,7 +3864,13 @@ def _row_to_record(row: sqlite3.Row) -> MemoryRecord: importance=row["importance"], surprise=row["surprise"], stability=row["stability"], access_count=row["access_count"], last_access=row["last_access"], valid_from=row["valid_from"], valid_to=row["valid_to"], + valid_to_recorded_at=( + row["valid_to_recorded_at"] + if "valid_to_recorded_at" in row.keys() else None + ), ingested_at=row["ingested_at"], expired_at=row["expired_at"], + subject_key=row["subject_key"] if "subject_key" in row.keys() else "", + claim_kind=row["claim_kind"] if "claim_kind" in row.keys() else "", pinned=bool(row["pinned"]), sensitivity=row["sensitivity"], provenance=_loads(row["provenance"], {}), ) @@ -2696,6 +3885,10 @@ def _row_to_edge(row: sqlite3.Row) -> Edge: weight=row["weight"], workspace_id=row["workspace_id"] if "workspace_id" in row.keys() else None, repo_id=row["repo_id"] if "repo_id" in row.keys() else None, valid_from=row["valid_from"], valid_to=row["valid_to"], + valid_to_recorded_at=( + row["valid_to_recorded_at"] + if "valid_to_recorded_at" in row.keys() else None + ), ingested_at=row["ingested_at"], expired_at=row["expired_at"], provenance=_loads(row["provenance"], {}), ) diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py index 9b3b9014..7f5e478c 100644 --- a/engraphis/core/sync.py +++ b/engraphis/core/sync.py @@ -56,7 +56,8 @@ # ── bundle format ───────────────────────────────────────────────────────────── SYNC_FORMAT = "engraphis-sync" -SYNC_VERSION = 1 +SYNC_VERSION = 2 +SYNC_ACCEPTED_VERSIONS = frozenset({1, 2}) # ── validation caps (untrusted bundle → clamp, don't trust) ─────────────────── MAX_MEMORIES = 200_000 @@ -90,7 +91,7 @@ _LWW_FIELDS = ( "title", "content", "summary", "keywords", "metadata", "mtype", "scope", "importance", "surprise", "sensitivity", "valid_from", "ingested_at", - "session_id", "provenance", + "session_id", "provenance", "subject_key", "claim_kind", ) @@ -135,6 +136,7 @@ def _label_tuple(rec: MemoryRecord) -> list: rec.title, rec.content, rec.summary, sorted(rec.keywords or []), _enum(rec.mtype), _enum(rec.scope), rec.importance, rec.surprise, rec.sensitivity, rec.valid_from, rec.session_id, + rec.subject_key, rec.claim_kind, json.dumps(rec.metadata or {}, sort_keys=True, default=str), json.dumps(rec.provenance or {}, sort_keys=True, default=str), ] @@ -156,6 +158,7 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: are taken from ``local`` here and never LWW-merged, so re-homing is never undone. """ winner = local if _version_key(local) >= _version_key(incoming) else incoming + valid_to, valid_to_recorded_at = _merge_closure(local, incoming) return MemoryRecord( id=local.id, # scope pointers are always local — never merged from the remote @@ -167,6 +170,7 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: mtype=winner.mtype, scope=winner.scope, importance=winner.importance, surprise=winner.surprise, sensitivity=winner.sensitivity, session_id=winner.session_id, provenance=dict(winner.provenance or {}), + subject_key=winner.subject_key, claim_kind=winner.claim_kind, valid_from=winner.valid_from, # ``ingested_at`` is a LWW field (_LWW_FIELDS), NOT a lattice field, and it is the # SECOND component of _version_key. Merging it as a min-lattice made @@ -180,12 +184,40 @@ def merge_record(local: MemoryRecord, incoming: MemoryRecord) -> MemoryRecord: # entirely from the winner. ingested_at=winner.ingested_at, # lattice fields: commutative joins (independent of the LWW winner) - valid_to=_min_nonnull(local.valid_to, incoming.valid_to), + valid_to=valid_to, expired_at=_min_nonnull(local.expired_at, incoming.expired_at), stability=max(local.stability, incoming.stability), access_count=max(local.access_count, incoming.access_count), last_access=_max_nonnull(local.last_access, incoming.last_access), pinned=bool(local.pinned or incoming.pinned), + valid_to_recorded_at=valid_to_recorded_at, + ) + + +def _merge_closure( + local: MemoryRecord, incoming: MemoryRecord, +) -> tuple[Optional[float], Optional[float]]: + """Join a world-time closure with the system-time at which it was learned. + + The earliest world-time closure wins. Its knowledge timestamp must travel + with it; independently learned equal closures use the earliest timestamp. + A missing timestamp is legacy v1 state whose closure was always visible, so + it remains ``None`` rather than being silently assigned a later time. + """ + if local.valid_to is None: + return incoming.valid_to, ( + incoming.valid_to_recorded_at if incoming.valid_to is not None else None + ) + if incoming.valid_to is None: + return local.valid_to, local.valid_to_recorded_at + if local.valid_to < incoming.valid_to: + return local.valid_to, local.valid_to_recorded_at + if incoming.valid_to < local.valid_to: + return incoming.valid_to, incoming.valid_to_recorded_at + if local.valid_to_recorded_at is None or incoming.valid_to_recorded_at is None: + return local.valid_to, None + return local.valid_to, min( + local.valid_to_recorded_at, incoming.valid_to_recorded_at ) @@ -228,7 +260,8 @@ def inherit_store_defaults(existing: MemoryRecord, incoming: MemoryRecord) -> Me def _signature(rec: MemoryRecord) -> str: """Fingerprint of everything sync persists — to tell 'changed' from 'no-op'.""" return _stable_hash(_label_tuple(rec) + [ - rec.valid_to, rec.expired_at, rec.ingested_at, rec.stability, + rec.valid_to, rec.valid_to_recorded_at, rec.expired_at, + rec.ingested_at, rec.stability, rec.access_count, rec.last_access, bool(rec.pinned), ]) @@ -244,8 +277,10 @@ def record_to_dict(rec: MemoryRecord) -> dict: "importance": rec.importance, "surprise": rec.surprise, "stability": rec.stability, "access_count": rec.access_count, "last_access": rec.last_access, "valid_from": rec.valid_from, "valid_to": rec.valid_to, + "valid_to_recorded_at": rec.valid_to_recorded_at, "ingested_at": rec.ingested_at, "expired_at": rec.expired_at, "pinned": bool(rec.pinned), "sensitivity": rec.sensitivity, + "subject_key": rec.subject_key, "claim_kind": rec.claim_kind, "provenance": rec.provenance or {}, } @@ -424,9 +459,12 @@ def dict_to_record(d: dict) -> Optional[MemoryRecord]: # (they are the version key's primary ordering / anti-poison defense). valid_from=_clamp_world_ts(d.get("valid_from")), valid_to=_clamp_world_ts(d.get("valid_to")), + valid_to_recorded_at=_clamp_ts(d.get("valid_to_recorded_at"), now), ingested_at=_clamp_ts(d.get("ingested_at"), now), expired_at=_clamp_ts(d.get("expired_at"), now), pinned=bool(d.get("pinned")), sensitivity=sens, + subject_key=_clamp_str(d.get("subject_key"), 512), + claim_kind=_clamp_str(d.get("claim_kind"), 256), provenance=_safe_json_obj(d.get("provenance")), ) @@ -514,7 +552,7 @@ def apply_bundle(self, bundle: Any, *, into_workspace: Optional[str] = None, raise SyncError("bundle is not an object") if bundle.get("format") != SYNC_FORMAT: raise SyncError("not an %s bundle" % SYNC_FORMAT) - if _as_int(bundle.get("version"), 0) != SYNC_VERSION: + if _as_int(bundle.get("version"), 0) not in SYNC_ACCEPTED_VERSIONS: raise SyncError("unsupported bundle version %r" % bundle.get("version")) src_device = bundle.get("device_id") @@ -664,6 +702,15 @@ def _apply_one(self, d: dict, rec, report: dict, accepted: dict, known: dict, # overwrite the local private row with a non-session scope either. report["rejected"] += 1 return + if existing is not None: + # Sync v1 bundles predate durable claim identity. Omission means + # "unknown to this peer", not an instruction to erase local keys. + if "subject_key" not in d: + rec.subject_key = existing.subject_key + if "claim_kind" not in d: + rec.claim_kind = existing.claim_kind + if "valid_to_recorded_at" not in d and rec.valid_to == existing.valid_to: + rec.valid_to_recorded_at = existing.valid_to_recorded_at if (existing is not None and only_repo_id is not None and existing.repo_id != only_repo_id): # The incoming row's claimed repo cannot re-home an existing memory from @@ -735,7 +782,9 @@ def _apply_links(self, link_dicts: list, report: dict, accepted: dict, pending = 0 existing_link = self.store.conn.execute( "SELECT layer, reason FROM mem_links " - "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? LIMIT 1", + "WHERE ((a=? AND b=?) OR (a=? AND b=?)) AND relation=? " + "AND valid_to IS NULL AND expired_at IS NULL " + "ORDER BY rowid DESC LIMIT 1", (a, b, b, a, rel), ).fetchone() if existing_link: diff --git a/engraphis/logging_setup.py b/engraphis/logging_setup.py index bfcf9926..81f4a1eb 100644 --- a/engraphis/logging_setup.py +++ b/engraphis/logging_setup.py @@ -14,6 +14,8 @@ import os import time +from engraphis.observability import redact, redact_json_value + _PLAIN_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" # Attributes present on every LogRecord — everything else came in via ``extra=``. @@ -31,13 +33,16 @@ def format(self, record: logging.LogRecord) -> str: + ".%03dZ" % (record.msecs,), "level": record.levelname, "logger": record.name, - "message": record.getMessage(), + "message": redact(record.getMessage()), } for key, value in record.__dict__.items(): if key not in _RESERVED and not key.startswith("_"): - payload[key] = value + # Use the same field-name-aware path as nested mappings: a bare + # ``extra={"refresh_credential": ...}`` otherwise has no adjacent + # text for the value-level credential recognizer to rely on. + payload[key] = redact_json_value({key: value})[key] if record.exc_info: - payload["exc_info"] = self.formatException(record.exc_info) + payload["exc_info"] = redact(self.formatException(record.exc_info)) return json.dumps(payload, ensure_ascii=False, default=str) diff --git a/engraphis/mcp_server.py b/engraphis/mcp_server.py index 78df7bce..07756801 100644 --- a/engraphis/mcp_server.py +++ b/engraphis/mcp_server.py @@ -47,8 +47,10 @@ workspace (or "default" only when none was supplied), the current repository name when known, and k=5. For every multi-step task, first call engraphis_start_session with the same workspace/repo plus the client name and task goal; retain -its session_id and use its bootstrap handoff. Recall before asking the user for information they -may already have provided. +its session_id and use its bootstrap handoff. For query-driven prompt context, prefer +engraphis_recall_context with the smallest sufficient token_budget; use engraphis_recall only +when complete memory bodies are explicitly needed. Recall before asking the user for information +they may already have provided. Store only durable facts, decisions with rationale, preferences, bug cause/fix pairs, and reusable procedures through engraphis_remember using the narrowest reusable scope. Never store credentials, @@ -168,8 +170,9 @@ def engraphis_remember( "lexical recall.")] = None, dedupe: Annotated[bool, Field(description="If true (default), check this against similar " "existing memories first: an exact restatement reinforces the existing " - "one instead of duplicating it, and a same-subject update supersedes the " - "old one (closed, not deleted) instead of leaving a contradiction. Set " + "one instead of duplicating it; a shared subject_key or strong joint " + "evidence can supersede the old one, while uncertain neighbors are " + "related without discarding either fact. Set " "false to force a plain insert (e.g. for recurring episodic log " "entries where repeats are meaningful).")] = True, source: Annotated[str, Field(description="Provenance: who/what produced this memory — " @@ -189,6 +192,16 @@ def engraphis_remember( retention_reason: Annotated[str, Field( description="Short explanation for the retention classification; do not repeat " "sensitive memory contents.", max_length=1_000)] = "", + valid_from: Annotated[Optional[float], Field( + description="Optional Unix timestamp for when this fact became true in world time. " + "Omit to use ingestion time.")] = None, + subject_key: Annotated[str, Field( + description="Optional stable claim subject (for example 'api.rate_limit'). " + "Matching keys make supersession safer and deterministic.", + max_length=1_000)] = "", + claim_kind: Annotated[str, Field( + description="Optional claim predicate/category (for example 'configured_value').", + max_length=200)] = "", ) -> str: """Store a memory so it can be recalled in later turns, sessions, or repos. @@ -200,7 +213,8 @@ def engraphis_remember( ``op`` is ``"add"`` (new), ``"noop"`` (matched an existing memory almost exactly — that one was reinforced, ``id`` points to it), or ``"invalidate"`` (superseded an existing memory on the same subject — see ``superseded`` for the old id(s); history - is preserved, never deleted). Returns ``"Error: "`` if validation fails. + is preserved, never deleted), or ``"relate"`` (kept both uncertain neighboring claims + and linked them). Returns ``"Error: "`` if validation fails. """ try: return _ok(service().remember( @@ -208,6 +222,8 @@ def engraphis_remember( mtype=mtype, scope=scope, title=title, importance=importance, keywords=keywords, source=source, trusted=trusted, kind=kind, retention_class=retention_class, retention_reason=retention_reason, + valid_from=valid_from, + subject_key=subject_key, claim_kind=claim_kind, resolve_conflicts=dedupe, )) except Exception as exc: # noqa: BLE001 - surface a safe, actionable message @@ -234,13 +250,34 @@ def engraphis_recall( mtypes: Annotated[Optional[List[str]], Field(description="Restrict to these memory types " "(semantic/episodic/procedural/working).")] = None, k: Annotated[int, Field(description="Max memories to return (1-50).", ge=1, le=50)] = 8, + as_of: Annotated[Optional[float], Field( + description="Compatibility alias for valid_at. If both are supplied they must " + "match.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp: return facts true then.")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp: return only facts Engraphis " + "had learned and not retired then.")] = None, + token_budget: Annotated[Optional[int], Field( + description="Hard packed-context budget under the named token counter (0-32768).", + ge=0, le=32_768)] = None, + retrieval_profile: Annotated[str, Field( + description="Retrieval profile: balanced (legacy hybrid), auto, lexical, graph, " + "or code. Auto is opt-in until benchmarks demonstrate a win.")] = "balanced", + response_mode: Annotated[str, Field( + description="full preserves legacy memory bodies; compact omits bodies already " + "represented in the packed context.")] = "full", + diagnostics: Annotated[bool, Field( + description="Include per-arm raw/normalized/fusion/rerank diagnostics.")] = False, ) -> str: """Retrieve the memories most relevant to a query (hybrid vector + lexical + graph). Call this before answering or acting when prior context would help — to avoid re-asking the user, to recover decisions/conventions, or to resume earlier work. - Successful calls reinforce returned memories and append a privacy-safe recall receipt, - so this retrieval surface is intentionally neither read-only nor idempotent. + Successful calls append a privacy-safe recall receipt but do not strengthen weak + neighbors merely because they were returned. Grounded recall reinforces cited + evidence; an explicit-use caller can opt into reinforcement through the Python API. + Because the receipt is stateful, this surface is neither read-only nor idempotent. Returns: str: JSON with ``{"query","count","context","memories":[{"id","title","content", @@ -250,12 +287,100 @@ def engraphis_recall( try: return _ok(service().recall( query, workspace=workspace, repo=repo, session_id=session_id, - mtypes=mtypes, k=k, + mtypes=mtypes, k=k, as_of=as_of, valid_at=valid_at, + known_at=known_at, token_budget=token_budget, + retrieval_profile=retrieval_profile, response_mode=response_mode, + diagnostics=diagnostics, )) except Exception as exc: # noqa: BLE001 return _err(exc) +@mcp.tool( + name="engraphis_recall_context", + annotations={"title": "Recall token-efficient context", "readOnlyHint": False, + "destructiveHint": False, "idempotentHint": False, + "openWorldHint": False}, +) +def engraphis_recall_context( + query: Annotated[str, Field(description="What prior context is needed.", + min_length=1, max_length=100_000)], + workspace: Annotated[Optional[str], Field(description="Restrict to this workspace.", + max_length=200)] = None, + repo: Annotated[Optional[str], Field(description="Restrict to this repo (requires " + "workspace).", max_length=200)] = None, + session_id: Annotated[Optional[str], Field( + description="Optional active session; includes its repo/workspace ancestors.")] = None, + mtypes: Annotated[Optional[List[str]], Field( + description="Optional memory types: semantic/episodic/procedural/working.")] = None, + k: Annotated[int, Field(description="Max candidate memories (1-50).", ge=1, le=50)] = 8, + token_budget: Annotated[int, Field( + description="Hard packed-context budget under the reported token counter.", + ge=0, le=32_768)] = 1024, + retrieval_profile: Annotated[str, Field( + description="balanced, auto, lexical, graph, or code.")] = "balanced", + as_of: Annotated[Optional[float], Field( + description="Compatibility alias for valid_at.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp.")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp.")] = None, + diagnostics: Annotated[bool, Field( + description="Include detailed retrieval scoring trace.")] = False, +) -> str: + """Return one hard-budget context plus compact source identities. + + This is the recommended agent path: unlike legacy full recall, it does not + repeat every complete memory body alongside the already-packed context. The + response includes exact accounting for the declared counter, omitted/packed + counts, and privacy-safe savings metadata. + """ + try: + payload = service().recall( + query, + workspace=workspace, + repo=repo, + session_id=session_id, + mtypes=mtypes, + k=k, + as_of=as_of, + valid_at=valid_at, + known_at=known_at, + token_budget=token_budget, + retrieval_profile=retrieval_profile, + response_mode="compact", + diagnostics=diagnostics, + intent="recall_context", + ) + by_id = { + str(source.get("id") or ""): source + for source in payload.pop("memories", []) + } + sources = [] + for ordinal, packed in enumerate(payload.pop("packed_sources", []), start=1): + detail = by_id.get(str(packed.get("id") or ""), {}) + source = { + "n": ordinal, + "id": packed.get("id"), + "tokens": packed.get("tokens"), + } + if detail.get("title"): + source["title"] = detail["title"] + provenance = detail.get("provenance") + if provenance: + source["provenance"] = provenance + if packed.get("truncated"): + source["truncated"] = True + reason = packed.get("reason") + if reason and reason not in {"full", "summary"}: + source["reason"] = reason + sources.append(source) + payload["sources"] = sources + return _ok(payload) + except Exception as exc: # noqa: BLE001 + return _err(exc) + + @mcp.tool( name="engraphis_recall_grounded", annotations={"title": "Grounded recall (cited answer, or abstain)", @@ -276,6 +401,9 @@ def engraphis_recall_grounded( mtypes: Annotated[Optional[List[str]], Field(description="Restrict to these memory types " "(semantic/episodic/procedural/working).")] = None, k: Annotated[int, Field(description="Max memories to consider (1-50).", ge=1, le=50)] = 8, + # These original parameters stay before all newly-added options. MCP clients use + # named fields, but established Python callers may invoke this decorated callable + # positionally. min_support: Annotated[Optional[float], Field(description="Absolute support floor 0..1 " "below which the tool abstains instead of answering. Omit for the " "default; raise it to demand stronger evidence (0 disables the abstain gate).", ge=0.0, @@ -283,6 +411,21 @@ def engraphis_recall_grounded( synthesize: Annotated[bool, Field(description="If true and an LLM is configured, " "synthesize cited prose; otherwise return the deterministic " "extractive answer.")] = False, + as_of: Annotated[Optional[float], Field( + description="Compatibility alias for valid_at.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp.")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp.")] = None, + token_budget: Annotated[Optional[int], Field( + description="Hard packed-context budget (0-32768).", ge=0, le=32_768)] = None, + retrieval_profile: Annotated[str, Field( + description="balanced, auto, lexical, graph, or code.")] = "balanced", + response_mode: Annotated[str, Field( + description="full includes citation bodies; compact omits bodies already present " + "in the cited answer.")] = "full", + diagnostics: Annotated[bool, Field( + description="Include detailed retrieval scoring trace.")] = False, ) -> str: """Answer a question *strictly from* stored memories, with citations — or abstain. @@ -312,8 +455,10 @@ def engraphis_recall_grounded( llm = None return _ok(service().grounded_recall( query, workspace=workspace, repo=repo, session_id=session_id, - mtypes=mtypes, k=k, - min_support=min_support, llm=llm, + mtypes=mtypes, k=k, as_of=as_of, valid_at=valid_at, + known_at=known_at, token_budget=token_budget, + retrieval_profile=retrieval_profile, response_mode=response_mode, + diagnostics=diagnostics, min_support=min_support, llm=llm, )) except Exception as exc: # noqa: BLE001 return _err(exc) @@ -341,6 +486,21 @@ def engraphis_answer( k: Annotated[int, Field(description="Max memories to consider (1-50).", ge=1, le=50)] = 8, min_support: Annotated[float, Field(description="Absolute support floor 0..1. Memories below this don't count as evidence.", ge=0.0, le=1.0)] = 0.25, synthesize: Annotated[bool, Field(description="If true, ask configured LLM for cited prose; otherwise deterministic/extractive.")] = False, + as_of: Annotated[Optional[float], Field( + description="Optional Unix timestamp for a point-in-time grounded answer. " + "Omit for now.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp (must match as_of if both are set).")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp.")] = None, + token_budget: Annotated[Optional[int], Field( + description="Hard packed-context budget (0-32768).", ge=0, le=32_768)] = None, + retrieval_profile: Annotated[str, Field( + description="balanced, auto, lexical, graph, or code.")] = "balanced", + response_mode: Annotated[str, Field( + description="full includes citation bodies; compact omits them.")] = "full", + diagnostics: Annotated[bool, Field( + description="Include detailed retrieval scoring trace.")] = False, ) -> str: """Backward-compatible alias for ``engraphis_recall_grounded``. @@ -349,6 +509,9 @@ def engraphis_answer( """ return engraphis_recall_grounded( query=query, workspace=workspace, repo=repo, session_id=None, mtypes=None, k=k, + as_of=as_of, valid_at=valid_at, known_at=known_at, + token_budget=token_budget, retrieval_profile=retrieval_profile, + response_mode=response_mode, diagnostics=diagnostics, min_support=min_support, synthesize=synthesize, ) @@ -769,6 +932,12 @@ def engraphis_search_code( max_length=200)], limit: Annotated[int, Field(description="Max symbols to return (1-50).", ge=1, le=50)] = 20, + as_of: Annotated[Optional[float], Field( + description="Compatibility alias for valid_at.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp.")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp.")] = None, ) -> str: """Find function/class/method definitions by name, with their callers — structural code search that costs far fewer tokens than grepping/reading whole files, and @@ -779,7 +948,10 @@ def engraphis_search_code( "signature","called_by":[{"src","file","line"}]}]}``. """ try: - return _ok(service().search_code(query, workspace=workspace, repo=repo, limit=limit)) + return _ok(service().search_code( + query, workspace=workspace, repo=repo, limit=limit, as_of=as_of, + valid_at=valid_at, known_at=known_at, + )) except Exception as exc: # noqa: BLE001 return _err(exc) @@ -800,6 +972,12 @@ def engraphis_code_path( min_length=1, max_length=200)], max_depth: Annotated[int, Field(description="Maximum graph hops (1-32).", ge=1, le=32)] = 8, + as_of: Annotated[Optional[float], Field( + description="Compatibility alias for valid_at.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp.")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp.")] = None, ) -> str: """Return the shortest best-effort path between two code nodes. @@ -810,6 +988,7 @@ def engraphis_code_path( try: return _ok(service().code_path( source, target, workspace=workspace, repo=repo, max_depth=max_depth, + as_of=as_of, valid_at=valid_at, known_at=known_at, )) except Exception as exc: # noqa: BLE001 return _err(exc) @@ -829,11 +1008,18 @@ def engraphis_code_impact( min_length=1, max_length=200)], repo: Annotated[str, Field(description="Indexed repo to analyze.", min_length=1, max_length=200)], + as_of: Annotated[Optional[float], Field( + description="Compatibility alias for valid_at.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp.")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp.")] = None, ) -> str: """Estimate affected symbols, callers, memories, graph communities, and risk.""" try: return _ok(service().code_impact( - changed_files, workspace=workspace, repo=repo, + changed_files, workspace=workspace, repo=repo, as_of=as_of, + valid_at=valid_at, known_at=known_at, )) except Exception as exc: # noqa: BLE001 return _err(exc) @@ -849,10 +1035,19 @@ def engraphis_export_code_graph( min_length=1, max_length=200)], repo: Annotated[str, Field(description="Indexed repo to export.", min_length=1, max_length=200)], + as_of: Annotated[Optional[float], Field( + description="Compatibility alias for valid_at.")] = None, + valid_at: Annotated[Optional[float], Field( + description="Optional world-time Unix timestamp.")] = None, + known_at: Annotated[Optional[float], Field( + description="Optional system-time Unix timestamp.")] = None, ) -> str: """Export portable graph JSON plus a human-readable Markdown report.""" try: - return _ok(service().export_code_graph(workspace=workspace, repo=repo)) + return _ok(service().export_code_graph( + workspace=workspace, repo=repo, as_of=as_of, + valid_at=valid_at, known_at=known_at, + )) except Exception as exc: # noqa: BLE001 return _err(exc) diff --git a/engraphis/observability.py b/engraphis/observability.py index 0f1221b2..89d7794e 100644 --- a/engraphis/observability.py +++ b/engraphis/observability.py @@ -5,6 +5,7 @@ import logging import os import re +from collections.abc import Mapping from datetime import datetime, timezone @@ -64,6 +65,32 @@ def redact(value: object) -> str: return _EMAIL.sub("[email]", text) +def redact_json_value(value: object, *, _depth: int = 0) -> object: + """Return a JSON-safe log value without preserving credential-bearing extras. + + A logger's ``extra`` mapping bypasses ``LogRecord.getMessage()``, so a formatter that + emits those fields directly needs the same redaction boundary as an ordinary message. + Keep modest structure for useful operational fields while bounding hostile/cyclic values. + """ + if _depth >= 8: + return "[redacted]" + if isinstance(value, Mapping): + result = {} + for key, item in value.items(): + name = str(key) + result[name] = ( + "[redacted]" + if re.fullmatch(_SENSITIVE_NAME, name.upper()) + else redact_json_value(item, _depth=_depth + 1) + ) + return result + if isinstance(value, (list, tuple, set, frozenset)): + return [redact_json_value(item, _depth=_depth + 1) for item in value] + if value is None or isinstance(value, (bool, int, float)): + return value + return redact(value) + + class RedactedJsonFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: payload = { diff --git a/engraphis/read_only_api.py b/engraphis/read_only_api.py index 98de2ea6..d6d64af9 100644 --- a/engraphis/read_only_api.py +++ b/engraphis/read_only_api.py @@ -20,6 +20,12 @@ class IntentRecallRequest(BaseModel): mtypes: Optional[list[str]] = None k: int = 8 as_of: Optional[float] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None + token_budget: Optional[int] = None + retrieval_profile: str = "balanced" + response_mode: str = "compact" + diagnostics: bool = False class CodePathRequest(BaseModel): @@ -28,12 +34,18 @@ class CodePathRequest(BaseModel): source: str target: str max_depth: int = 8 + as_of: Optional[float] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None class CodeImpactRequest(BaseModel): workspace: str repo: str changed_files: list[str] + as_of: Optional[float] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None def create_read_only_app(service: Optional[MemoryService] = None, *, @@ -72,9 +84,19 @@ def health(): @app.get("/recall") def recall(query: str, workspace: Optional[str] = None, - repo: Optional[str] = None, k: int = 8): + repo: Optional[str] = None, k: int = 8, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", + response_mode: str = "compact", + diagnostics: bool = False): return run( svc.recall, query, workspace=workspace, repo=repo, k=k, + as_of=as_of, valid_at=valid_at, known_at=known_at, + token_budget=token_budget, retrieval_profile=retrieval_profile, + response_mode=response_mode, diagnostics=diagnostics, reinforce=False, intent="http_read_only", record_receipt=False, ) @@ -83,43 +105,63 @@ def intent_recall(req: IntentRecallRequest): return run( svc.intent_recall, req.query, intent=req.intent, workspace=req.workspace, repo=req.repo, mtypes=req.mtypes, - k=req.k, as_of=req.as_of, reinforce=False, record_receipt=False, + k=req.k, as_of=req.as_of, valid_at=req.valid_at, + known_at=req.known_at, token_budget=req.token_budget, + retrieval_profile=req.retrieval_profile, + response_mode=req.response_mode, diagnostics=req.diagnostics, + reinforce=False, record_receipt=False, ) @app.get("/graph") def graph(workspace: str, limit: int = 2_000, layers: Optional[str] = None, - include_code: bool = False, repo: Optional[str] = None): + include_code: bool = False, repo: Optional[str] = None, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): selected = None if layers is None else [ value.strip() for value in layers.split(",") if value.strip() ] return run( svc.graph, workspace=workspace, limit=limit, layers=selected, include_code=include_code, repo=repo, backfill=False, + as_of=as_of, valid_at=valid_at, known_at=known_at, ) @app.get("/code/search") - def code_search(query: str, workspace: str, repo: str, limit: int = 20): + def code_search(query: str, workspace: str, repo: str, limit: int = 20, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): return run( svc.search_code, query, workspace=workspace, repo=repo, limit=limit, + as_of=as_of, valid_at=valid_at, known_at=known_at, ) @app.post("/code/path") def code_path(req: CodePathRequest): return run( svc.code_path, req.source, req.target, workspace=req.workspace, - repo=req.repo, max_depth=req.max_depth, + repo=req.repo, max_depth=req.max_depth, as_of=req.as_of, + valid_at=req.valid_at, known_at=req.known_at, ) @app.post("/code/impact") def code_impact(req: CodeImpactRequest): return run( svc.code_impact, req.changed_files, - workspace=req.workspace, repo=req.repo, + workspace=req.workspace, repo=req.repo, as_of=req.as_of, + valid_at=req.valid_at, known_at=req.known_at, ) @app.get("/code/export") - def code_export(workspace: str, repo: str): - return run(svc.export_code_graph, workspace=workspace, repo=repo) + def code_export(workspace: str, repo: str, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): + return run( + svc.export_code_graph, workspace=workspace, repo=repo, + as_of=as_of, valid_at=valid_at, known_at=known_at, + ) @app.get("/receipts") def receipts(workspace: str, limit: int = 100): diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py index ea92e2c3..24a6e321 100644 --- a/engraphis/routes/v2_api.py +++ b/engraphis/routes/v2_api.py @@ -252,8 +252,11 @@ def _mem(m: dict) -> dict: "importance": m.get("importance"), "valid_from": m.get("valid_from"), "valid_to": m.get("valid_to"), + "valid_to_recorded_at": m.get("valid_to_recorded_at"), "expired_at": m.get("expired_at"), "ingested_at": m.get("ingested_at"), + "subject_key": m.get("subject_key") or "", + "claim_kind": m.get("claim_kind") or "", "provenance": m.get("provenance") or {}, } @@ -264,7 +267,9 @@ def _is_embedder_mismatch(exc) -> bool: return "not aligned" in message or ("256" in message and "384" in message) -def _keyword_search(ws, q, limit=20): +def _keyword_search(ws, q, limit=20, *, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): """Non-semantic fallback: match memories by keyword (title/content LIKE) so the Recall/Why/Timeline tabs still return results when the embedder is unavailable.""" import json as _json @@ -276,11 +281,28 @@ def _keyword_search(ws, q, limit=20): row = conn.execute("SELECT id FROM workspaces WHERE name=?", (ws,)).fetchone() if row is None: return [] + # Match the public Recall contract even if semantic retrieval cannot run. + # A model-dimension mismatch must degrade retrieval quality, never silently + # turn a historical request into a present-time data leak. + if as_of is not None and valid_at is not None and float(as_of) != float(valid_at): + raise ValidationError("as_of and valid_at must match when both are supplied") + world_anchor = float(valid_at if valid_at is not None else as_of) if ( + valid_at is not None or as_of is not None + ) else time.time() + system_anchor = float(known_at) if known_at is not None else time.time() sql = ("SELECT id, scope, mtype, title, content, summary, pinned, importance, " - "valid_from, valid_to, provenance FROM memories WHERE workspace_id=? " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, " + "subject_key, claim_kind, provenance FROM memories WHERE workspace_id=? " "AND COALESCE(scope, 'workspace')!='session' " - "AND valid_to IS NULL AND expired_at IS NULL") - args = [row["id"]] + "AND (valid_from IS NULL OR valid_from<=?) " + "AND (valid_to IS NULL OR ? 2][:6] if terms: sql += " AND (" + " OR ".join(["title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\'" for _ in terms]) + ")" @@ -301,7 +323,11 @@ def _prov(pp): "content": r["content"] or r["summary"] or "", "memory_type": r["mtype"] or "semantic", "scope": r["scope"] or "", "pinned": bool(r["pinned"]), "importance": r["importance"], "valid_from": r["valid_from"], - "valid_to": r["valid_to"], "provenance": _prov(r["provenance"])} for r in rows] + "valid_to": r["valid_to"], + "valid_to_recorded_at": r["valid_to_recorded_at"], + "ingested_at": r["ingested_at"], "expired_at": r["expired_at"], + "subject_key": r["subject_key"] or "", "claim_kind": r["claim_kind"] or "", + "provenance": _prov(r["provenance"])} for r in rows] # ── health / bootstrap ──────────────────────────────────────────────────────── @@ -551,8 +577,9 @@ def _llm_is_verified(provider: str, model: str) -> bool: @router.get("/llm/status") def llm_status(): """Report the configured LLM provider/model/key presence and the active extractor, - plus a ready-to-paste .env snippet for the dashboard's "Connect your LLM" card. - Never returns the API key or custom provider endpoint — only whether each is set.""" + retention-supervision mode, and a ready-to-paste .env snippet for the dashboard's + "Connect your LLM" card. Never returns the API key or custom provider endpoint — + only whether each is set.""" provider = settings.llm_provider or "openai" model = settings.llm_model or _LLM_DEFAULT_MODELS.get(provider, "") key_set = bool(settings.llm_api_key) @@ -564,6 +591,7 @@ def llm_status(): "custom_base_url_configured": bool(settings.llm_base_url), "extractor": settings.extractor, "extractor_enabled": _extractor_enabled(), + "retention_supervisor": settings.retention_supervisor, "auto_extract": bool(settings.llm_auto_extract), "configured": key_set and bool(model), "working": verified, @@ -925,11 +953,20 @@ def stats(workspace: Optional[str] = None): # ── recall / search ─────────────────────────────────────────────────────────── @router.get("/recall") def recall(q: str = Query(...), workspace: Optional[str] = None, k: int = 8, - mtype: Optional[str] = None): + mtype: Optional[str] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, known_at: Optional[float] = None, + token_budget: Optional[int] = Query(default=None, ge=0, le=32_768), + retrieval_profile: str = "balanced", response_mode: str = "full", + diagnostics: bool = False): ws = workspace or _default_ws() mtypes = [mtype] if mtype else None try: - out = service().recall(q, workspace=ws, k=k, mtypes=mtypes, reinforce=False) + out = service().recall( + q, workspace=ws, k=k, mtypes=mtypes, as_of=as_of, + valid_at=valid_at, known_at=known_at, reinforce=False, + token_budget=token_budget, retrieval_profile=retrieval_profile, + response_mode=response_mode, diagnostics=diagnostics, + ) except ValidationError: logger.info("dashboard recall request rejected") raise _invalid_request() from None @@ -937,13 +974,22 @@ def recall(q: str = Query(...), workspace: Optional[str] = None, k: int = 8, if not _is_embedder_mismatch(exc): logger.error("dashboard recall failed (%s)", type(exc).__name__) raise HTTPException(status_code=500, detail={"error": "internal server error"}) - mems = _keyword_search(ws, q, k) + mems = _keyword_search( + ws, q, k, as_of=as_of, valid_at=valid_at, known_at=known_at, + ) return {"query": q, "workspace": ws, "count": len(mems), "context": "", "memories": mems, "mode": "keyword", "note": "Keyword match — install sentence-transformers for semantic search."} - return {"query": q, "workspace": ws, "count": out.get("count", 0), - "context": out.get("context", ""), "mode": "semantic", - "memories": [_mem(m) for m in out.get("memories", [])]} + payload = dict(out) + payload.update({ + "query": q, + "workspace": ws, + "count": out.get("count", 0), + "context": out.get("context", ""), + "mode": "semantic", + "memories": [_mem(m) for m in out.get("memories", [])], + }) + return payload class _AnswerReq(BaseModel): @@ -953,6 +999,13 @@ class _AnswerReq(BaseModel): k: int = Field(default=8, ge=1, le=50) max_citations: int = Field(default=5, ge=1, le=50) min_support: Optional[float] = Field(default=None, ge=0.0, le=1.0) + as_of: Optional[float] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None + token_budget: Optional[int] = Field(default=None, ge=0, le=32_768) + retrieval_profile: str = "balanced" + response_mode: str = "full" + diagnostics: bool = False @router.post("/answer") @@ -971,6 +1024,13 @@ def answer(req: _AnswerReq): workspace=ws, repo=req.repo, k=req.k, + as_of=req.as_of, + valid_at=req.valid_at, + known_at=req.known_at, + token_budget=req.token_budget, + retrieval_profile=req.retrieval_profile, + response_mode=req.response_mode, + diagnostics=req.diagnostics, max_citations=req.max_citations, min_support=req.min_support, ) @@ -1219,6 +1279,9 @@ class _RememberReq(BaseModel): dedupe: bool = True retention_class: Optional[str] = None retention_reason: str = "" + valid_from: Optional[float] = None + subject_key: str = "" + claim_kind: str = "" @router.post("/remember") @@ -1228,7 +1291,9 @@ def remember(req: _RememberReq): importance=req.importance, keywords=req.keywords, metadata=req.metadata, source=req.source, trusted=req.trusted, resolve_conflicts=req.dedupe, retention_class=req.retention_class, - retention_reason=req.retention_reason) + retention_reason=req.retention_reason, + valid_from=req.valid_from, + subject_key=req.subject_key, claim_kind=req.claim_kind) class _IntentRememberReq(BaseModel): @@ -1242,6 +1307,7 @@ class _IntentRememberReq(BaseModel): metadata: Optional[dict] = None retention_class: Optional[str] = None retention_reason: str = "" + valid_from: Optional[float] = None @router.post("/intent/remember") @@ -1253,6 +1319,7 @@ def intent_remember(req: _IntentRememberReq): title=req.title, mtype=req.mtype, scope=req.scope, importance=req.importance, metadata=req.metadata, retention_class=req.retention_class, retention_reason=req.retention_reason, + valid_from=req.valid_from, ) @@ -1283,6 +1350,12 @@ class _IntentRecallReq(BaseModel): mtypes: Optional[list] = None k: int = 8 as_of: Optional[float] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None + token_budget: Optional[int] = Field(default=None, ge=0, le=32_768) + retrieval_profile: str = "balanced" + response_mode: str = "compact" + diagnostics: bool = False @router.post("/intent/recall") @@ -1291,6 +1364,9 @@ def intent_recall(req: _IntentRecallReq): service().intent_recall, req.query, intent=req.intent, workspace=req.workspace or _default_ws(), repo=req.repo, mtypes=req.mtypes, k=req.k, as_of=req.as_of, + valid_at=req.valid_at, known_at=req.known_at, + token_budget=req.token_budget, retrieval_profile=req.retrieval_profile, + response_mode=req.response_mode, diagnostics=req.diagnostics, ) @@ -1369,8 +1445,9 @@ def ready(): # ── workspace export (local, free) ──────────────────────────────────────────── @router.get("/export") -def export(workspace: Optional[str] = None, signed: bool = False): - """Full bi-temporal workspace dump (memories + sessions + audit). +def export(workspace: Optional[str] = None, signed: bool = False, + canonical: bool = False): + """Portable v2 workspace dump, including temporal graph/code evidence and receipts. This is the free local data-portability path. ``signed=true`` was never implemented, so it must not claim availability in Engraphis Cloud either. @@ -1383,7 +1460,12 @@ def export(workspace: Optional[str] = None, signed: bool = False): "alternative": "/export", }) ws = workspace or _default_ws() - return _run(service().export_workspace, workspace=ws, recovery=True) + return _run( + service().export_workspace, + workspace=ws, + recovery=True, + canonical=canonical, + ) # ── automated maintenance (Pro) ─────────────────────────────────────────────── @@ -1572,7 +1654,9 @@ def graph(workspace: Optional[str] = None, limit: int = 2000, layers: Optional[str] = None, include_code: bool = False, repo: Optional[str] = None, full: bool = False, connected_only: bool = False, - as_of: Optional[float] = None): + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): """Entity-relation network for a workspace — vis-network-ready nodes/edges plus type counts, top-connected, and connectivity stats. @@ -1591,7 +1675,7 @@ def graph(workspace: Optional[str] = None, limit: int = 2000, service().graph, workspace=ws, limit=limit, layers=selected, include_code=include_code, repo=repo, backfill=False, full=full, connected_only=connected_only, - as_of=as_of, + as_of=as_of, valid_at=valid_at, known_at=known_at, ) @@ -1615,6 +1699,8 @@ def graph_scene(workspace: Optional[str] = None, level: str = "overview", entity_types: Optional[str] = None, memory_types: Optional[str] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, time_from: Optional[float] = None, time_to: Optional[float] = None, depth: int = Query(default=1, ge=0, le=2), @@ -1647,7 +1733,8 @@ def graph_scene(workspace: Optional[str] = None, level: str = "overview", center_id=center_id, system_id=system_id, seeds=_graph_csv(seeds), repo=repo, layers=_graph_csv(layers), relations=_graph_csv(relations), entity_types=_graph_csv(entity_types), memory_types=_graph_csv(memory_types), - as_of=as_of, time_from=time_from, time_to=time_to, depth=depth, + as_of=as_of, valid_at=valid_at, known_at=known_at, + time_from=time_from, time_to=time_to, depth=depth, min_support=min_support, min_confidence=min_confidence, include_weak_cooccurrence=weak_cooccurrence, include_code=code_enabled, node_limit=node_limit, edge_limit=edge_limit, @@ -1660,6 +1747,8 @@ def graph_suggest(q: str = "", query: Optional[str] = None, repo: Optional[str] = None, memory_types: Optional[str] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, time_from: Optional[float] = None, time_to: Optional[float] = None, include_weak_cooccurrence: bool = False, @@ -1668,7 +1757,8 @@ def graph_suggest(q: str = "", query: Optional[str] = None, return _run( service().graph_suggest, query if query is not None else q, workspace=ws, repo=repo, memory_types=_graph_csv(memory_types), - as_of=as_of, time_from=time_from, time_to=time_to, + as_of=as_of, valid_at=valid_at, known_at=known_at, + time_from=time_from, time_to=time_to, include_weak_cooccurrence=include_weak_cooccurrence, limit=limit, ) @@ -1678,6 +1768,8 @@ def graph_entity(canonical_id: str, workspace: Optional[str] = None, repo: Optional[str] = None, memory_types: Optional[str] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, time_from: Optional[float] = None, time_to: Optional[float] = None, include_weak_cooccurrence: bool = True): @@ -1685,6 +1777,7 @@ def graph_entity(canonical_id: str, workspace: Optional[str] = None, return _run( service().graph_entity, canonical_id, workspace=ws, repo=repo, memory_types=_graph_csv(memory_types), as_of=as_of, + valid_at=valid_at, known_at=known_at, time_from=time_from, time_to=time_to, include_weak_cooccurrence=include_weak_cooccurrence, ) @@ -1692,17 +1785,22 @@ def graph_entity(canonical_id: str, workspace: Optional[str] = None, @router.get("/graph/entities/{canonical_id}/memories") def graph_entity_memories(canonical_id: str, workspace: Optional[str] = None, - as_of: Optional[float] = None): + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): """Bounded evidence cards for one graph node, without rebuilding the full inspector.""" ws = workspace or _require_ws() return _run( - service().graph_entity_evidence, canonical_id, workspace=ws, as_of=as_of, + service().graph_entity_evidence, canonical_id, workspace=ws, + as_of=as_of, valid_at=valid_at, known_at=known_at, ) @router.get("/graph/path") def graph_path(source: str, target: str, workspace: Optional[str] = None, repo: Optional[str] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, memory_types: Optional[str] = None, time_from: Optional[float] = None, time_to: Optional[float] = None, @@ -1712,7 +1810,8 @@ def graph_path(source: str, target: str, workspace: Optional[str] = None, ws = workspace or _require_ws() return _run( service().graph_path, source, target, workspace=ws, repo=repo, - as_of=as_of, memory_types=_graph_csv(memory_types), + as_of=as_of, valid_at=valid_at, known_at=known_at, + memory_types=_graph_csv(memory_types), time_from=time_from, time_to=time_to, max_hops=max_hops, max_visits=max_visits, include_weak_cooccurrence=include_weak_cooccurrence, @@ -1810,9 +1909,13 @@ def code_index(req: _CodeIndexReq): @router.get("/code/search") -def code_search(query: str, workspace: str, repo: str, limit: int = 20): +def code_search(query: str, workspace: str, repo: str, limit: int = 20, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): return _run( service().search_code, query, workspace=workspace, repo=repo, limit=limit, + as_of=as_of, valid_at=valid_at, known_at=known_at, ) @@ -1822,13 +1925,17 @@ class _CodePathReq(BaseModel): source: str target: str max_depth: int = 8 + as_of: Optional[float] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None @router.post("/code/path") def code_path(req: _CodePathReq): return _run( service().code_path, req.source, req.target, workspace=req.workspace, - repo=req.repo, max_depth=req.max_depth, + repo=req.repo, max_depth=req.max_depth, as_of=req.as_of, + valid_at=req.valid_at, known_at=req.known_at, ) @@ -1836,19 +1943,29 @@ class _CodeImpactReq(BaseModel): workspace: str repo: str changed_files: list[str] + as_of: Optional[float] = None + valid_at: Optional[float] = None + known_at: Optional[float] = None @router.post("/code/impact") def code_impact(req: _CodeImpactReq): return _run( service().code_impact, req.changed_files, - workspace=req.workspace, repo=req.repo, + workspace=req.workspace, repo=req.repo, as_of=req.as_of, + valid_at=req.valid_at, known_at=req.known_at, ) @router.get("/code/export") -def code_export(workspace: str, repo: str): - return _run(service().export_code_graph, workspace=workspace, repo=repo) +def code_export(workspace: str, repo: str, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None): + return _run( + service().export_code_graph, workspace=workspace, repo=repo, + as_of=as_of, valid_at=valid_at, known_at=known_at, + ) # ── license ─────────────────────────────────────────────────────────────────── @@ -2137,12 +2254,18 @@ def _normalized_features(values: object, plan: str) -> list: ``known_features`` even if a future server release adds a key this build predates. """ + allowed = set(entitled_features(plan)) if not isinstance(values, (list, tuple)): return entitled_features(plan) granted = {str(item).strip().lower() for item in values if isinstance(item, str)} if "automation" in granted: granted.update(_AUTOMATION_FEATURES) - return sorted(granted & set(_FEATURE_LABELS)) + # The payload is authoritative for a *subset* of the customer's plan grants, but it + # must never escalate them. In particular a stale or malformed Pro response that + # still lists ``team`` used to unlock the Team UI even though the plan had already + # changed. The server still authorizes every operation, but presentation must be + # conservative too. Unknown future keys remain hidden as before. + return sorted(granted & allowed & set(_FEATURE_LABELS)) def _session_entitlement() -> dict: diff --git a/engraphis/service.py b/engraphis/service.py index 77f86a0c..1965e52a 100644 --- a/engraphis/service.py +++ b/engraphis/service.py @@ -26,6 +26,7 @@ import time import threading from collections import Counter, OrderedDict +from dataclasses import asdict from functools import wraps from pathlib import Path from typing import Any, Optional @@ -41,7 +42,13 @@ from engraphis.core.graph_layers import normalize_graph_layer from engraphis.core.ids import new_id as make_id from engraphis.core.interfaces import Edge, GraphLayer, MemoryType, Node, Scope, SearchFilter -from engraphis.core.store import _loads, _merge_edge_provenance, normalize_entity_name +from engraphis.core.retrieval_policy import RETRIEVAL_PROFILES +from engraphis.core.store import ( + _loads, + _merge_edge_provenance, + _public_receipt_row, + normalize_entity_name, +) from engraphis.graphdata import build_graph_payload, empty_graph logger = logging.getLogger("engraphis.service") @@ -54,6 +61,8 @@ MAX_KEYWORD_CHARS = 128 MAX_METADATA_BYTES = 16_384 MAX_K = 50 +MAX_TOKEN_BUDGET = 32_768 +RESPONSE_MODES = frozenset({"full", "compact"}) MAX_CONTEXT_TASK_CHARS = 10_000 MAX_AGENT_STATE_CHARS = 20_000 # import_folder/import_files (SECURITY.md §5 — reads/accepts local-content by path or @@ -178,6 +187,25 @@ def _graph_entity_visibility_sql(entity_alias: str, *, at: Optional[float] = Non _NAME_RE = re.compile(r"^[A-Za-z0-9._\-/ ]{1,%d}$" % MAX_NAME_CHARS) _PRINCIPAL_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,%d}$" % MAX_NAME_CHARS) _PRINCIPAL_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+$") +_RECEIPT_ID_RE = re.compile(r"^rcpt_[0-9ABCDEFGHJKMNPQRSTVWXYZ]{26}$") +_RECEIPT_HASH_RE = re.compile(r"^[0-9a-f]{64}$") +_RECEIPT_VERIFICATION_ERRORS = frozenset({ + "hash_mismatch", + "payload_mismatch", + "payload_schema_invalid", + "sequence_mismatch", + "chain_break", + "chain_root_count", + "chain_cycle", + "chain_fork", + "chain_disconnected", + "missing_anchor", + "anchor_count_mismatch", + "anchor_head_mismatch", + "anchor_integrity_error", + "expected_head_mismatch", + "expected_count_mismatch", +}) class ValidationError(ValueError): @@ -422,6 +450,38 @@ def _enum(value: Any, enum_cls, field: str): raise ValidationError(f"{field} must be one of: {allowed}") +def _optional_timestamp(value: Any, *, field: str) -> Optional[float]: + """Validate an optional Unix timestamp at the shared transport boundary.""" + if value is None: + return None + if isinstance(value, bool): + raise ValidationError(f"{field} must be a finite timestamp") + try: + timestamp = float(value) + except (TypeError, ValueError) as exc: + raise ValidationError(f"{field} must be a finite timestamp") from exc + if not math.isfinite(timestamp): + raise ValidationError(f"{field} must be a finite timestamp") + return timestamp + + +def _temporal_anchors(*, as_of: Any = None, valid_at: Any = None, + known_at: Any = None) -> tuple[Optional[float], Optional[float], Optional[float]]: + """Normalize the public bi-temporal aliases once for non-recall reads. + + Recall performs the same validation inline for backwards-compatible error + ordering. Direct code-graph reads use this helper so they cannot silently + diverge from recall's ``as_of``/``valid_at`` contract. + """ + as_of_value = _optional_timestamp(as_of, field="as_of") + valid_value = _optional_timestamp(valid_at, field="valid_at") + known_value = _optional_timestamp(known_at, field="known_at") + if as_of_value is not None and valid_value is not None and as_of_value != valid_value: + raise ValidationError("as_of and valid_at must match when both are supplied") + valid_value = valid_value if valid_value is not None else as_of_value + return as_of_value, valid_value, known_value + + def _write_scope(value: Any, *, repo: Optional[str], session_id: Optional[str]) -> Scope: """Resolve and validate the structural scope of a write. @@ -649,6 +709,7 @@ def _graph_scene_valid_until(self, workspace_id: str, at: float) -> float: @classmethod def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, + embed_revision: Optional[str] = None, embed_dim: int = 384, vector_backend: str = "auto", rerank_model: Optional[str] = None, allowed_workspaces: Optional[list] = None, @@ -678,7 +739,8 @@ def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, from engraphis.backends.encrypted_db import connector_from_env connect = connector_from_env() engine = MemoryEngine.create( - db_path, embed_model=embed_model, embed_dim=embed_dim, + db_path, embed_model=embed_model, embed_revision=embed_revision, + embed_dim=embed_dim, vector_backend=vector_backend, rerank_model=rerank_model, extractor=extractor, graph_extractor=graph_extractor, retention_supervisor=retention_supervisor, connect=connect, @@ -895,9 +957,12 @@ def remember(self, content: str, *, workspace: str, repo: Optional[str] = None, source: str = "agent", trusted: bool = True, kind: Optional[str] = None, resolve_conflicts: bool = True, retention_class: Optional[str] = None, - retention_reason: str = "") -> dict: + retention_reason: str = "", + valid_from: Optional[float] = None, + subject_key: str = "", claim_kind: str = "") -> dict: """Store one memory. Returns its id, resolved scope, and the resolution - outcome (``op``: add/noop/invalidate — see ``MemoryEngine.remember_with_resolution``). + outcome (``op``: add/noop/invalidate/relate — see + ``MemoryEngine.remember_with_resolution``). """ content = _clean_text(content, field="content", max_chars=MAX_CONTENT_CHARS) title = _clean_text(title, field="title", max_chars=MAX_TITLE_CHARS, required=False) @@ -908,6 +973,13 @@ def remember(self, content: str, *, workspace: str, repo: Optional[str] = None, sc = _write_scope(scope, repo=rp, session_id=session_id) kws = _clean_keywords(keywords) meta = _clean_metadata(metadata) + valid_from = _optional_timestamp(valid_from, field="valid_from") + subject_key = _clean_text( + subject_key, field="subject_key", max_chars=MAX_TITLE_CHARS, required=False + ) + claim_kind = _clean_text( + claim_kind, field="claim_kind", max_chars=MAX_NAME_CHARS, required=False + ) retention = None if retention_class: label = _clean_text( @@ -957,9 +1029,13 @@ def remember(self, content: str, *, workspace: str, repo: Optional[str] = None, content, workspace_id=wid, repo_id=rid, session_id=session_id, mtype=mt, scope=sc, title=title, importance=importance, keywords=kws, metadata={**meta, "provenance": provenance}, + valid_from=valid_from, + subject_key=subject_key, claim_kind=claim_kind, resolve_conflicts=bool(resolve_conflicts), ) except ValueError as exc: + if str(exc).startswith("valid_from "): + raise ValidationError(str(exc)) from exc if session_id and str(exc) in { f"no session with id '{session_id}'", "session_id does not belong to that workspace/repo", @@ -971,10 +1047,12 @@ def remember(self, content: str, *, workspace: str, repo: Optional[str] = None, "id": result["id"], "workspace": ws, "repo": rp, "scope": sc.value, "mtype": mt.value, "stored": True, "op": result["op"], } - if result["op"] in ("noop", "invalidate"): + if result["op"] in ("noop", "invalidate", "relate"): out["resolution"] = result.get("reason", "") if result["op"] == "invalidate": out["superseded"] = result["superseded"] + if result["op"] == "relate": + out["related_to"] = result.get("related_to") out["receipt"] = self.store.record_receipt( "remember", workspace_id=wid, repo_id=rid or "", actor=provenance["source"], target_count=1, status=result["op"], @@ -1056,11 +1134,13 @@ def intent_remember(self, text: str, *, workspace: str, importance: float = 0.0, metadata: Optional[dict] = None, retention_class: Optional[str] = None, - retention_reason: str = "") -> dict: + retention_reason: str = "", + valid_from: Optional[float] = None) -> dict: out = self.remember( text, workspace=workspace, repo=repo, title=title, mtype=mtype, scope=scope, importance=importance, metadata=metadata, retention_class=retention_class, retention_reason=retention_reason, + valid_from=valid_from, ) return {"operation": "remember", **out} @@ -1076,7 +1156,13 @@ def intent_recall(self, query: str, *, intent: str = "recall", workspace: Optional[str] = None, repo: Optional[str] = None, mtypes: Optional[list] = None, k: int = 8, as_of: Optional[float] = None, - reinforce: bool = True, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", + response_mode: str = "full", + diagnostics: bool = False, + reinforce: bool = False, record_receipt: bool = True) -> dict: intent_clean = _clean_text( intent, field="intent", max_chars=80, required=False @@ -1094,13 +1180,17 @@ def intent_recall(self, query: str, *, intent: str = "recall", }.get(normalized) out = self.recall( query, workspace=workspace, repo=repo, mtypes=mtypes, k=k, - as_of=as_of, intent=intent_clean, graph_layers=layers, + as_of=as_of, valid_at=valid_at, known_at=known_at, + token_budget=token_budget, retrieval_profile=retrieval_profile, + response_mode=response_mode, diagnostics=diagnostics, + intent=intent_clean, graph_layers=layers, reinforce=reinforce, record_receipt=record_receipt, ) response = {"operation": "recall", "intent": intent_clean, **out} if normalized in {"locate_code", "code"} and workspace and repo: response["code"] = self.search_code( - query, workspace=workspace, repo=repo, limit=k + query, workspace=workspace, repo=repo, limit=k, + as_of=as_of, valid_at=valid_at, known_at=known_at, ) elif normalized in {"explain", "why"} and workspace: response["explanation"] = self.why( @@ -1567,8 +1657,14 @@ def recall(self, query: str, *, workspace: Optional[str] = None, repo: Optional[str] = None, session_id: Optional[str] = None, mtypes: Optional[list] = None, k: int = 8, as_of: Optional[float] = None, - reinforce: bool = True, intent: str = "recall", + valid_at: Optional[float] = None, + known_at: Optional[float] = None, + reinforce: bool = False, intent: str = "recall", graph_layers: Optional[list] = None, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", + response_mode: str = "full", + diagnostics: bool = False, record_receipt: bool = True) -> dict: """Retrieve the most relevant memories for ``query`` within scope.""" query = _clean_text(query, field="query", max_chars=MAX_CONTENT_CHARS) @@ -1582,6 +1678,27 @@ def recall(self, query: str, *, workspace: Optional[str] = None, [_enum(layer, GraphLayer, "graph_layer") for layer in graph_layers] if graph_layers else None ) + as_of = _optional_timestamp(as_of, field="as_of") + valid_at = _optional_timestamp(valid_at, field="valid_at") + known_at = _optional_timestamp(known_at, field="known_at") + if as_of is not None and valid_at is not None and as_of != valid_at: + raise ValidationError("as_of and valid_at must match when both are supplied") + valid_at = valid_at if valid_at is not None else as_of + try: + token_budget = ( + self.engine.recall_engine.token_budget + if token_budget is None else int(token_budget) + ) + except (TypeError, ValueError): + raise ValidationError("token_budget must be an integer") + token_budget = max(0, min(MAX_TOKEN_BUDGET, token_budget)) + retrieval_profile = str(retrieval_profile or "balanced").strip().casefold() + if retrieval_profile not in RETRIEVAL_PROFILES: + choices = ", ".join(sorted(RETRIEVAL_PROFILES)) + raise ValidationError(f"retrieval_profile must be one of: {choices}") + response_mode = str(response_mode or "full").strip().casefold() + if response_mode not in RESPONSE_MODES: + raise ValidationError("response_mode must be one of: compact, full") # A configured workspace binding or a bound dashboard user must never do a # workspace-less (global) recall — either case represents a tenant boundary. @@ -1595,22 +1712,32 @@ def recall(self, query: str, *, workspace: Optional[str] = None, ws = self._clean_ws(workspace) wid = self._lookup_workspace(ws) if wid is None: - return {"query": query, "count": 0, "context": "", "memories": [], - "note": f"no workspace named '{ws}' yet"} + return _empty_recall( + query, token_budget=token_budget, response_mode=response_mode, + retrieval_profile=retrieval_profile, valid_at=valid_at, + known_at=known_at, note=f"no workspace named '{ws}' yet", + ) if repo: rp = _clean_name(repo, field="repo") rid = self._lookup_repo(wid, rp) if rid is None: - return {"query": query, "count": 0, "context": "", "memories": [], - "note": f"no repo named '{rp}' in workspace '{ws}' yet"} + return _empty_recall( + query, token_budget=token_budget, response_mode=response_mode, + retrieval_profile=retrieval_profile, valid_at=valid_at, + known_at=known_at, + note=f"no repo named '{rp}' in workspace '{ws}' yet", + ) if session_id: sid = _clean_text( session_id, field="session_id", max_chars=MAX_NAME_CHARS ) session = self.store.get_session(sid) if session is None: - return {"query": query, "count": 0, "context": "", "memories": [], - "note": f"no session with id '{sid}'"} + return _empty_recall( + query, token_budget=token_budget, response_mode=response_mode, + retrieval_profile=retrieval_profile, valid_at=valid_at, + known_at=known_at, note=f"no session with id '{sid}'", + ) if session["workspace_id"] != wid or ( rid is not None and session.get("repo_id") != rid): raise ValidationError("session_id does not belong to that workspace/repo") @@ -1621,30 +1748,74 @@ def recall(self, query: str, *, workspace: Optional[str] = None, result = self.engine.recall_engine.recall( query, - _filter(wid, rid, mts, as_of, layers, session_id=sid), + _filter( + wid, rid, mts, as_of, layers, session_id=sid, + valid_at=valid_at, known_at=known_at, + ), k=k, reinforce=reinforce, + token_budget=token_budget, + retrieval_profile=retrieval_profile, + diagnostics=bool(diagnostics), ) memories = [] for chunk in result.chunks: - item = dict(chunk) - arm = item.get("arm") or "hybrid" - item["why_recalled"] = ( - f"Matched by {arm} retrieval; fused score " - f"{float(item.get('score') or 0.0):.3f}, retention " - f"{float(item.get('retention') or 0.0):.3f}." - ) + if response_mode == "compact": + item = { + key: chunk.get(key) + for key in ( + "id", "title", "scope", "mtype", "repo_id", "score", "arm" + ) + } + item["provenance"] = _compact_provenance(chunk.get("provenance")) + else: + item = dict(chunk) + arm = item.get("arm") or "hybrid" + item["why_recalled"] = ( + f"Matched by {arm} retrieval; fused score " + f"{float(item.get('score') or 0.0):.3f}, retention " + f"{float(item.get('retention') or 0.0):.3f}." + ) memories.append(item) + usage = asdict(result.usage) if result.usage is not None else { + "budget_tokens": token_budget, + "context_tokens": 0, + "source_tokens": 0, + "saved_tokens": 0, + "savings_ratio": 0.0, + "packed_count": 0, + "omitted_count": 0, + "token_counter": "unknown", + } + packed_sources = [{ + "id": packed.id, + "tokens": packed.tokens, + "truncated": packed.truncated, + "reason": packed.reason, + } for packed in result.packed_chunks] out = { "query": query, "count": result.count, "context": result.context, "memories": memories, + "packed_sources": packed_sources, + "usage": usage, + "valid_at": result.valid_at, + "known_at": result.known_at, + "historical": result.historical, + "retrieval_profile": result.retrieval_profile, + "response_mode": response_mode, } + if diagnostics: + out["retrieval_trace"] = result.retrieval_trace or [] if record_receipt: out["receipt"] = self.store.record_receipt( "recall", workspace_id=wid or "", repo_id=rid or "", actor="agent", target_count=result.count, status="ok", metadata={"intent": str(intent or "recall")[:80], "k": k, "result_count": result.count, - "graph_layers": [layer.value for layer in layers] if layers else []}, + "graph_layers": [layer.value for layer in layers] if layers else [], + "retrieval_profile": result.retrieval_profile, + "response_mode": response_mode, + "historical": result.historical, + "token_usage": usage}, ) return out @@ -1652,8 +1823,14 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, repo: Optional[str] = None, session_id: Optional[str] = None, mtypes: Optional[list] = None, k: int = 8, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, min_support: Optional[float] = None, - max_citations: int = 5, llm=None) -> dict: + max_citations: int = 5, llm=None, + token_budget: Optional[int] = None, + retrieval_profile: str = "balanced", + response_mode: str = "full", + diagnostics: bool = False) -> dict: """Grounded recall: an answer built strictly from retrieved memories, with ``[n]`` citations and an explicit abstain when evidence is insufficient (``core.grounded``). This path is offline/deterministic (extractive answer) — no @@ -1672,6 +1849,27 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, except (TypeError, ValueError): raise ValidationError("max_citations must be an integer") max_citations = max(1, min(MAX_K, max_citations)) + as_of = _optional_timestamp(as_of, field="as_of") + valid_at = _optional_timestamp(valid_at, field="valid_at") + known_at = _optional_timestamp(known_at, field="known_at") + if as_of is not None and valid_at is not None and as_of != valid_at: + raise ValidationError("as_of and valid_at must match when both are supplied") + valid_at = valid_at if valid_at is not None else as_of + try: + token_budget = ( + self.engine.recall_engine.token_budget + if token_budget is None else int(token_budget) + ) + except (TypeError, ValueError): + raise ValidationError("token_budget must be an integer") + token_budget = max(0, min(MAX_TOKEN_BUDGET, token_budget)) + retrieval_profile = str(retrieval_profile or "balanced").strip().casefold() + if retrieval_profile not in RETRIEVAL_PROFILES: + choices = ", ".join(sorted(RETRIEVAL_PROFILES)) + raise ValidationError(f"retrieval_profile must be one of: {choices}") + response_mode = str(response_mode or "full").strip().casefold() + if response_mode not in RESPONSE_MODES: + raise ValidationError("response_mode must be one of: compact, full") if min_support is not None: try: min_support = float(min_support) @@ -1692,25 +1890,35 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, ws = self._clean_ws(workspace) wid = self._lookup_workspace(ws) if wid is None: - return {"query": query, "grounded": False, "abstained": True, - "answer": "", "support": 0.0, "citations": [], - "reason": f"no workspace named '{ws}' yet"} + return _empty_grounded( + query, reason=f"no workspace named '{ws}' yet", + token_budget=token_budget, response_mode=response_mode, + retrieval_profile=retrieval_profile, valid_at=valid_at, + known_at=known_at, + ) if repo: rp = _clean_name(repo, field="repo") rid = self._lookup_repo(wid, rp) if rid is None: - return {"query": query, "grounded": False, "abstained": True, - "answer": "", "support": 0.0, "citations": [], - "reason": f"no repo named '{rp}' in workspace '{ws}' yet"} + return _empty_grounded( + query, + reason=f"no repo named '{rp}' in workspace '{ws}' yet", + token_budget=token_budget, response_mode=response_mode, + retrieval_profile=retrieval_profile, valid_at=valid_at, + known_at=known_at, + ) if session_id: sid = _clean_text( session_id, field="session_id", max_chars=MAX_NAME_CHARS ) session = self.store.get_session(sid) if session is None: - return {"query": query, "grounded": False, "abstained": True, - "answer": "", "support": 0.0, "citations": [], - "reason": f"no session with id '{sid}'"} + return _empty_grounded( + query, reason=f"no session with id '{sid}'", + token_budget=token_budget, response_mode=response_mode, + retrieval_profile=retrieval_profile, valid_at=valid_at, + known_at=known_at, + ) if session["workspace_id"] != wid or ( rid is not None and session.get("repo_id") != rid): raise ValidationError("session_id does not belong to that workspace/repo") @@ -1721,16 +1929,31 @@ def grounded_recall(self, query: str, *, workspace: Optional[str] = None, ans = self.engine.grounded_recall( query, workspace_id=wid, repo_id=rid, session_id=sid, mtypes=mts, - as_of=as_of, k=k, llm=llm, min_support=min_support, - max_citations=max_citations, + as_of=as_of, valid_at=valid_at, known_at=known_at, + k=k, llm=llm, min_support=min_support, + max_citations=max_citations, token_budget=token_budget, + retrieval_profile=retrieval_profile, diagnostics=bool(diagnostics), ) out = {"query": query, **ans.to_dict()} + out["response_mode"] = response_mode + if response_mode == "compact": + compact_citations = [] + for citation in out.get("citations") or []: + item = dict(citation) + item.pop("content", None) + item["provenance"] = _compact_provenance(item.get("provenance")) + compact_citations.append(item) + out["citations"] = compact_citations out["receipt"] = self.store.record_receipt( - "recall", workspace_id=wid or "", repo_id=rid or "", actor="agent", + "grounded_recall", workspace_id=wid or "", repo_id=rid or "", actor="agent", target_count=len(out.get("citations") or []), status="grounded" if out.get("grounded") else "abstained", metadata={"intent": "grounded", "grounded": bool(out.get("grounded")), - "citations": len(out.get("citations") or [])}, + "citations": len(out.get("citations") or []), + "retrieval_profile": out.get("retrieval_profile"), + "response_mode": response_mode, + "historical": bool(out.get("historical")), + "token_usage": out.get("usage") or {}}, ) return out @@ -2095,21 +2318,37 @@ def index_repo(self, *, workspace: str, repo: str, root_path: str, ) return out - def search_code(self, query: str, *, workspace: str, repo: str, limit: int = 20) -> dict: + def search_code(self, query: str, *, workspace: str, repo: str, limit: int = 20, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> dict: + """Search code symbols and their memory bridges at one bi-temporal point. + + ``as_of`` is retained as the legacy alias for ``valid_at``. Keeping the + anchors on the direct code endpoint matters as much as on hybrid recall: + callers otherwise receive a historical memory answer accompanied by present-day + symbols and code-memory evidence. + """ if not repo: raise ValidationError("repo is required to search code") query = _clean_text(query, field="query", max_chars=MAX_CONTENT_CHARS) wid, rid = self._require_scope(workspace, repo) limit = max(1, min(MAX_K, int(limit))) + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) return self.engine.search_code( query, repo_id=rid, limit=limit, flt=SearchFilter( - workspace_id=wid, repo_id=rid, include_ancestors=True + workspace_id=wid, repo_id=rid, include_ancestors=True, + as_of=as_of, valid_at=valid_at, known_at=known_at, ), ) def code_path(self, source: str, target: str, *, workspace: str, repo: str, - max_depth: int = 8) -> dict: + max_depth: int = 8, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> dict: if not repo: raise ValidationError("repo is required for a code path query") source = _clean_text(source, field="source", max_chars=500) @@ -2119,33 +2358,51 @@ def code_path(self, source: str, target: str, *, workspace: str, repo: str, max_depth = max(1, min(32, int(max_depth))) except (TypeError, ValueError): raise ValidationError("max_depth must be an integer") + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) return self.engine.code_path( source, target, repo_id=rid, max_depth=max_depth, flt=SearchFilter( - workspace_id=wid, repo_id=rid, include_ancestors=True + workspace_id=wid, repo_id=rid, include_ancestors=True, + as_of=as_of, valid_at=valid_at, known_at=known_at, ), ) - def code_impact(self, changed_files: list, *, workspace: str, repo: str) -> dict: + def code_impact(self, changed_files: list, *, workspace: str, repo: str, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> dict: if not repo: raise ValidationError("repo is required for impact analysis") files = _clean_string_list( changed_files, field="changed_files", max_items=2_000, max_chars=4_000 ) wid, rid = self._require_scope(workspace, repo) + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) return self.engine.analyze_impact( files, repo_id=rid, flt=SearchFilter( - workspace_id=wid, repo_id=rid, include_ancestors=True + workspace_id=wid, repo_id=rid, include_ancestors=True, + as_of=as_of, valid_at=valid_at, known_at=known_at, ), ) - def export_code_graph(self, *, workspace: str, repo: str) -> dict: + def export_code_graph(self, *, workspace: str, repo: str, + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> dict: if not repo: raise ValidationError("repo is required to export a code graph") wid, rid = self._require_scope(workspace, repo) + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) flt = SearchFilter( - workspace_id=wid, repo_id=rid, include_ancestors=True + workspace_id=wid, repo_id=rid, include_ancestors=True, + as_of=as_of, valid_at=valid_at, known_at=known_at, ) graph = self.engine.export_code_graph(repo_id=rid, flt=flt) return { @@ -2156,6 +2413,9 @@ def export_code_graph(self, *, workspace: str, repo: str) -> dict: "graph_html": self.engine.code_graph_html( repo_id=rid, payload=graph, flt=flt ), + "valid_at": valid_at, + "known_at": known_at, + "historical": flt.historical, } # ── inspection (powers the Memory Inspector UI) ───────────────────────────── @@ -2396,6 +2656,12 @@ def delete_workspace(self, workspace: str, *, actor: str = "user") -> dict: f"OR memory_id IN {msub} OR symbol_id IN {ssub}", (wid, wid, wid), ) + c.execute( + "DELETE FROM memory_entities WHERE workspace_id=? " + f"OR memory_id IN {msub} " + "OR entity_id IN (SELECT id FROM entities WHERE workspace_id=?)", + (wid, wid, wid), + ) c.execute(f"DELETE FROM mem_fts WHERE id IN {msub}", (wid,)) c.execute(f"DELETE FROM mem_vectors WHERE id IN {msub}", (wid,)) try: @@ -2413,6 +2679,8 @@ def delete_workspace(self, workspace: str, *, actor: str = "user") -> dict: c.execute(f"DELETE FROM symbols WHERE repo_id IN {rsub}", (wid,)) c.execute("DELETE FROM repos WHERE workspace_id=?", (wid,)) c.execute("DELETE FROM jobs WHERE workspace_id=?", (wid,)) + c.execute("DELETE FROM operation_receipts WHERE workspace_id=?", (wid,)) + c.execute("DELETE FROM receipt_chain_heads WHERE workspace_id=?", (wid,)) # Entity/edge delete triggers may have recreated this generation row. c.execute("DELETE FROM graph_index_state WHERE workspace_id=?", (wid,)) c.execute("DELETE FROM workspaces WHERE id=?", (wid,)) @@ -2596,6 +2864,73 @@ def _new_repo(old_repo_id): (new_id, wid_dst, old_id), ) + # Rehome the persisted sparse graph index alongside its memory/entity + # endpoints. Entity folding can make two live incidences equivalent; retain + # the duplicate as closed history instead of violating the partial unique + # index or deleting evidence. + incidence_closed_at = time.time() + source_incidence = [dict(row) for row in c.execute( + "SELECT * FROM memory_entities WHERE workspace_id=? ORDER BY id", + (wid_src,), + )] + for incidence in source_incidence: + mapped_entity = entity_remap.get( + incidence["entity_id"], incidence["entity_id"] + ) + mapped_repo = _new_repo(incidence["repo_id"]) + live = incidence["valid_to"] is None and incidence["expired_at"] is None + duplicate = None + if live: + duplicate = c.execute( + "SELECT id, confidence, valid_from, ingested_at " + "FROM memory_entities WHERE id<>? AND memory_id=? " + "AND entity_id=? AND source_kind=? " + "AND valid_to IS NULL AND expired_at IS NULL LIMIT 1", + ( + incidence["id"], incidence["memory_id"], mapped_entity, + incidence["source_kind"], + ), + ).fetchone() + if duplicate is None: + c.execute( + "UPDATE memory_entities SET workspace_id=?, repo_id=?, entity_id=? " + "WHERE id=?", + (wid_dst, mapped_repo, mapped_entity, incidence["id"]), + ) + continue + valid_values = [ + value for value in ( + duplicate["valid_from"], incidence["valid_from"] + ) if value is not None + ] + known_values = [ + value for value in ( + duplicate["ingested_at"], incidence["ingested_at"] + ) if value is not None + ] + c.execute( + "UPDATE memory_entities SET confidence=?, valid_from=?, ingested_at=? " + "WHERE id=?", + ( + max( + float(duplicate["confidence"] or 0.0), + float(incidence["confidence"] or 0.0), + ), + min(valid_values) if valid_values else None, + min(known_values) if known_values else None, + duplicate["id"], + ), + ) + c.execute( + "UPDATE memory_entities SET workspace_id=?, repo_id=?, entity_id=?, " + "valid_to=?, valid_to_recorded_at=?, expired_at=? WHERE id=?", + ( + wid_dst, mapped_repo, mapped_entity, + incidence_closed_at, incidence_closed_at, + incidence_closed_at, incidence["id"], + ), + ) + # 3) Edges: relabel workspace/repo, remapping any entity ids folded in step 2. # When a live source edge collides with an existing live target edge (same # src/dst/relation/layer/repo), merge metadata instead of violating the @@ -2667,15 +3002,16 @@ def _new_repo(old_repo_id): # orphaned live evidence on a non-live edge, mirroring how # Store._deduplicate_live_edges() closes retired supports. c.execute( - "UPDATE edge_supports SET valid_to=?, expired_at=? " + "UPDATE edge_supports SET valid_to=?, valid_to_recorded_at=?, " + "expired_at=? " "WHERE edge_id=? AND valid_to IS NULL AND expired_at IS NULL", - (closed_at, closed_at, ed["id"])) + (closed_at, closed_at, closed_at, ed["id"])) # Bi-temporally close the source edge. src_prov["canonical_deduplicated_into"] = target["id"] c.execute( - "UPDATE edges SET valid_to=?, expired_at=?, " + "UPDATE edges SET valid_to=?, valid_to_recorded_at=?, expired_at=?, " "provenance=? WHERE id=?", - (closed_at, closed_at, + (closed_at, closed_at, closed_at, json.dumps(src_prov, ensure_ascii=False), ed["id"])) else: c.execute( @@ -2694,7 +3030,14 @@ def _new_repo(old_repo_id): f"WHERE workspace_id=? AND repo_id IS ?", (wid_dst, _new_repo(b["repo_id"]), wid_src, b["repo_id"])) - # 5) The source workspace is now empty — drop it. + # 5) Receipt payload hashes bind their original workspace scope digest and chain + # predecessor. Re-homing them would either forge that evidence or fork the target + # chain, so remove the source-only ledger with the source workspace. The merge's + # target-scoped audit entry below remains as the durable governance record. + c.execute("DELETE FROM operation_receipts WHERE workspace_id=?", (wid_src,)) + c.execute("DELETE FROM receipt_chain_heads WHERE workspace_id=?", (wid_src,)) + + # The source workspace is now empty — drop it. c.execute("DELETE FROM graph_index_state WHERE workspace_id=?", (wid_src,)) c.execute("DELETE FROM workspaces WHERE id=?", (wid_src,)) self.store.audit(actor, "workspace_merge", wid_dst, f"{src} ({int(n_mem)} memories) -> {dst}") @@ -2783,20 +3126,25 @@ def copy_workspace(self, source: str, new_name: Optional[str] = None, *, symbol_remap[s["id"]] = nsid c.execute( "INSERT INTO symbols(id, repo_id, kind, name, fqname, file, span, signature, " - "docstring, lang, exported, content_hash, embedding_ref, updated_at) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "docstring, lang, exported, content_hash, embedding_ref, updated_at, " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (nsid, nrid, s["kind"], s["name"], s["fqname"], s["file"], s["span"], s["signature"], s["docstring"], s["lang"], s["exported"], - s["content_hash"], s["embedding_ref"], s["updated_at"])) + s["content_hash"], s["embedding_ref"], s["updated_at"], + s["valid_from"], s["valid_to"], s["valid_to_recorded_at"], + s["ingested_at"], s["expired_at"])) for ce in [dict(x) for x in c.execute( "SELECT * FROM code_edges WHERE repo_id=?", (r["id"],))]: c.execute( - "INSERT INTO code_edges(id, repo_id, src, dst, relation, layer, file, line) " - "VALUES (?,?,?,?,?,?,?,?)", + "INSERT INTO code_edges(id, repo_id, src, dst, relation, layer, file, line, " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (ids.new_id("edge"), nrid, symbol_remap.get(ce["src"], ce["src"]), symbol_remap.get(ce["dst"], ce["dst"]), ce["relation"], ce["layer"] or "entity", - ce["file"], ce["line"])) + ce["file"], ce["line"], ce["valid_from"], ce["valid_to"], + ce["valid_to_recorded_at"], ce["ingested_at"], ce["expired_at"])) def _new_repo(old_repo_id): return repo_remap.get(old_repo_id, old_repo_id) if old_repo_id is not None else None @@ -2835,12 +3183,12 @@ def _new_repo(old_repo_id): edge_remap[ed["id"]] = new_edge_id c.execute( "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " - "weight, valid_from, valid_to, ingested_at, expired_at, provenance) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "weight, valid_from, valid_to, valid_to_recorded_at, ingested_at, " + "expired_at, provenance) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (new_edge_id, wid_dst, _new_repo(ed["repo_id"]), entity_remap.get(ed["src"], ed["src"]), entity_remap.get(ed["dst"], ed["dst"]), ed["relation"], ed["layer"] or "semantic", ed["weight"], - ed["valid_from"], ed["valid_to"], ed["ingested_at"], + ed["valid_from"], ed["valid_to"], ed["valid_to_recorded_at"], ed["ingested_at"], ed["expired_at"], ed["provenance"])) # 4) Sessions, cloned with fresh ids (memories/events below repoint at these). @@ -2899,20 +3247,30 @@ def walk(item): return json.dumps(walk(value), ensure_ascii=False, separators=(",", ":")) + def _remap_memory_ids_in_text(raw: Any) -> str: + text = str(raw or "") + return re.sub( + r"mem_[0-9ABCDEFGHJKMNPQRSTVWXYZ]{26}", + lambda match: memory_remap.get(match.group(0), match.group(0)), + text, + ) + for m in source_memories: nmid = memory_remap[m["id"]] c.execute( "INSERT INTO memories (id, workspace_id, repo_id, session_id, scope, mtype, " "title, content, summary, keywords, metadata, importance, surprise, stability, " - "access_count, last_access, valid_from, valid_to, ingested_at, expired_at, " - "pinned, sensitivity, provenance, sort_order) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "access_count, last_access, valid_from, valid_to, valid_to_recorded_at, " + "ingested_at, expired_at, subject_key, claim_kind, pinned, sensitivity, " + "provenance, sort_order) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (nmid, wid_dst, _new_repo(m["repo_id"]), session_remap.get(m["session_id"]), m["scope"], m["mtype"], m["title"], m["content"], m["summary"], m["keywords"], _remap_json_memory_ids(m["metadata"]), m["importance"], m["surprise"], m["stability"], m["access_count"], m["last_access"], m["valid_from"], m["valid_to"], - m["ingested_at"], m["expired_at"], m["pinned"], m["sensitivity"], + m["valid_to_recorded_at"], m["ingested_at"], m["expired_at"], + m["subject_key"], m["claim_kind"], m["pinned"], m["sensitivity"], _remap_json_memory_ids(m["provenance"]), m["sort_order"])) fts_row = c.execute( "SELECT title, content, keywords FROM mem_fts WHERE id=?", (m["id"],)).fetchone() @@ -2959,11 +3317,12 @@ def walk(item): ) c.execute( "INSERT INTO edge_supports(edge_id, memory_id, source_kind, confidence, " - "valid_from, valid_to, ingested_at, expired_at, provenance) " - "VALUES (?,?,?,?,?,?,?,?,?)", + "valid_from, valid_to, valid_to_recorded_at, ingested_at, " + "expired_at, provenance) VALUES (?,?,?,?,?,?,?,?,?,?)", (new_edge_id, new_memory_id, support["source_kind"], support["confidence"], support["valid_from"], support["valid_to"], - support["ingested_at"], support["expired_at"], support_provenance), + support["valid_to_recorded_at"], support["ingested_at"], + support["expired_at"], support_provenance), ) if not source_supports: try: @@ -2975,22 +3334,55 @@ def walk(item): new_edge_id, source_edge["relation"], fallback_provenance, valid_from=source_edge["valid_from"], valid_to=source_edge["valid_to"], + valid_to_recorded_at=source_edge["valid_to_recorded_at"], ingested_at=source_edge["ingested_at"], expired_at=source_edge["expired_at"], ) + # Persisted sparse memory↔entity incidence is a first-class retrieval index, + # not disposable cache. Clone it only after both endpoint maps exist so the + # copied workspace has graph-recall parity without retaining source ids. + for incidence in [dict(row) for row in c.execute( + "SELECT * FROM memory_entities WHERE workspace_id=? ORDER BY id", + (wid_src,), + )]: + new_memory_id = memory_remap.get(incidence["memory_id"]) + new_entity_id = entity_remap.get(incidence["entity_id"]) + if new_memory_id is None or new_entity_id is None: + continue + c.execute( + "INSERT INTO memory_entities(" + "id, memory_id, entity_id, workspace_id, repo_id, source_kind, confidence, " + "valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at, " + "provenance) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + ids.new_id("edge"), new_memory_id, new_entity_id, wid_dst, + _new_repo(incidence["repo_id"]), incidence["source_kind"], + incidence["confidence"], incidence["valid_from"], + incidence["valid_to"], incidence["valid_to_recorded_at"], + incidence["ingested_at"], incidence["expired_at"], + _remap_json_memory_ids(incidence["provenance"]), + ), + ) + if memory_remap: old_ids = list(memory_remap.keys()) marks = ",".join("?" for _ in old_ids) for ln in [dict(x) for x in c.execute( - f"SELECT a, b, relation, layer, reason, created_at FROM mem_links " + f"SELECT a, b, relation, layer, reason, created_at, valid_from, valid_to, " + f"valid_to_recorded_at, ingested_at, expired_at FROM mem_links " f"WHERE a IN ({marks}) AND b IN ({marks})", old_ids + old_ids)]: c.execute( - "INSERT INTO mem_links(a, b, relation, layer, reason, created_at) " - "VALUES (?,?,?,?,?,?)", + "INSERT INTO mem_links(" + "a, b, relation, layer, reason, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?)", ( memory_remap[ln["a"]], memory_remap[ln["b"]], - ln["relation"], ln["layer"], ln["reason"], ln["created_at"], + ln["relation"], ln["layer"], + _remap_memory_ids_in_text(ln["reason"]), ln["created_at"], + ln["valid_from"], ln["valid_to"], ln["valid_to_recorded_at"], + ln["ingested_at"], ln["expired_at"], ), ) @@ -3008,11 +3400,15 @@ def walk(item): continue c.execute( "INSERT INTO code_memory_links(id, repo_id, symbol_id, memory_id, " - "relation, confidence, created_at) VALUES (?,?,?,?,?,?,?)", + "relation, confidence, created_at, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", ( ids.new_id("edge"), repo_remap[link["repo_id"]], new_symbol, new_memory, link["relation"], - link["confidence"], link["created_at"], + link["confidence"], link["created_at"], link["valid_from"], + link["valid_to"], link["valid_to_recorded_at"], + link["ingested_at"], link["expired_at"], ), ) @@ -3252,58 +3648,645 @@ def verify_receipts(self, *, workspace: str, expected_head: str = "", raise ValidationError("expected_count must be an integer") if expected_count < 0: raise ValidationError("expected_count must be non-negative") - return self.store.verify_receipts( - workspace_id=wid, - expected_head=expected_head, - expected_count=expected_count, + return self._safe_receipt_verification( + self.store.verify_receipts( + workspace_id=wid, + expected_head=expected_head, + expected_count=expected_count, + ) ) + @staticmethod + def _redacted_receipt_value(value: Any) -> str: + raw = value if isinstance(value, str) else str(value or "") + return "redacted_sha256:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() + + @classmethod + def _safe_receipt_id(cls, value: Any, *, allow_empty: bool = False) -> str: + raw = value if isinstance(value, str) else str(value or "") + if allow_empty and not raw: + return "" + if _RECEIPT_ID_RE.fullmatch(raw): + return raw + return cls._redacted_receipt_value(raw) + + @classmethod + def _safe_receipt_hash(cls, value: Any, *, allow_empty: bool = False) -> str: + raw = value if isinstance(value, str) else str(value or "") + if allow_empty and not raw: + return "" + if _RECEIPT_HASH_RE.fullmatch(raw): + return raw + return cls._redacted_receipt_value(raw) + + @classmethod + def _safe_receipt_verification(cls, value: Any) -> dict: + """Project Store verification onto a fixed, content-free public schema.""" + raw = value if isinstance(value, dict) else {} + errors: list[dict] = [] + raw_errors = raw.get("errors") + if isinstance(raw_errors, list): + for item in raw_errors: + item = item if isinstance(item, dict) else {} + index = item.get("index") + if type(index) is not int or index < 0: + index = 0 + error = item.get("error") + if ( + not isinstance(error, str) + or error not in _RECEIPT_VERIFICATION_ERRORS + ): + error = cls._redacted_receipt_value(error) + errors.append({ + "index": index, + "id": cls._safe_receipt_id( + item.get("id"), allow_empty=True + ), + "error": error, + }) + count = raw.get("count") + if type(count) is not int or count < 0: + count = 0 + return { + "valid": raw.get("valid") is True and not errors, + "count": count, + "head": cls._safe_receipt_hash( + raw.get("head"), allow_empty=True + ), + "anchored": raw.get("anchored") is True, + "errors": errors, + } + def export_receipts(self, *, workspace: str) -> dict: - """Export only public receipt payloads and chain hashes.""" - out = self.receipt_log(workspace=workspace, limit=10_000) - out["verification"] = self.verify_receipts(workspace=workspace) - return out + """Export every public receipt payload and chain hash. + + ``receipt_log`` is deliberately a bounded inspection view. An export must not + silently inherit that 10,000-row ceiling because the omitted prefix is part of + both the chain and its independent count/head anchor. + """ + conn = self.store.conn + owns_transaction = not conn.transaction_owned_by_current_thread() + if owns_transaction: + conn.execute("BEGIN") + try: + wid, _ = self._require_scope(workspace, None) + result = { + "format": "engraphis-receipts/1", + "workspace_digest": hashlib.sha256(wid.encode("utf-8")).hexdigest()[:24], + "entries": self._complete_receipt_rows(wid), + "complete": True, + "verification": self._safe_receipt_verification( + self.store.verify_receipts(workspace_id=wid) + ), + } + if owns_transaction and conn.transaction_owned_by_current_thread(): + conn.commit() + return result + except BaseException: + if owns_transaction and conn.transaction_owned_by_current_thread(): + conn.rollback() + raise + + def _complete_receipt_rows(self, workspace_id: str) -> list[dict]: + """Return the complete receipt chain in predecessor order. + + Invalid/tampered payloads are represented instead of being dropped. That keeps + export counts honest and lets the verification result explain the corruption, + without reflecting arbitrary database text into a supposedly privacy-safe export. + """ + rows = list(self.store._receipt_chain_state(workspace_id)["rows"]) + return [_public_receipt_row(dict(row)) for row in rows] + + def export_workspace(self, *, workspace: str, recovery: bool = False, + canonical: bool = False) -> dict: + """Return one internally consistent portable workspace snapshot. + + A caller-owned transaction is never committed or rolled back here. Otherwise a + read transaction spans every constituent table and the receipt verification, so a + concurrent writer cannot produce a canonical digest of mutually impossible states. + """ + conn = self.store.conn + owns_transaction = not conn.transaction_owned_by_current_thread() + if owns_transaction: + conn.execute("BEGIN") + try: + result = self._export_workspace_snapshot( + workspace=workspace, + recovery=recovery, + canonical=canonical, + ) + if owns_transaction and conn.transaction_owned_by_current_thread(): + conn.commit() + return result + except BaseException: + if owns_transaction and conn.transaction_owned_by_current_thread(): + conn.rollback() + raise + + def _export_workspace_snapshot(self, *, workspace: str, recovery: bool = False, + canonical: bool = False) -> dict: + """Portable dump of durable workspace state, including bi-temporal history. - def export_workspace(self, *, workspace: str, recovery: bool = False) -> dict: - """Full bi-temporal dump of one workspace — memories (live *and* superseded), - sessions, and the audit trail. The compliance story in one artifact: nothing is - ever silently deleted, and the export proves it. Scope-checked like any other - read. Raw owner data portability is part of the local core; hosted signed and - formatted compliance reports are separate Cloud features.""" + Version 2 expands the original memory/session/audit export to the durable graph, + code, evidence, incidence, event, link, and receipt tables required to reconstruct + the workspace. Regenerable search indexes and process-local maintenance state are + explicitly disclosed as omitted. Authenticated non-admin callers receive shared + records plus only their own session-private records; every derivative reference is + filtered through that same boundary. + """ + del recovery # compatibility flag; local portability itself has no plan gate wid, _ = self._require_scope(workspace, None) conn = self.store.conn - user = _authenticated_principal() - if user is None: - memory_visibility = "" - session_visibility = "" - visibility_params: list[Any] = [] + principal = _authenticated_principal() + principal_scoped = principal is not None and principal.get("role") != "admin" + + workspace_row = dict(conn.execute( + "SELECT * FROM workspaces WHERE id=?", (wid,) + ).fetchone()) + repos = [dict(row) for row in conn.execute( + "SELECT * FROM repos WHERE workspace_id=? ORDER BY id", (wid,) + ).fetchall()] + repo_ids = {str(row["id"]) for row in repos} + + all_sessions = [dict(row) for row in conn.execute( + "SELECT * FROM sessions WHERE workspace_id=? ORDER BY id", (wid,) + ).fetchall()] + if principal_scoped: + sessions = [ + row for row in all_sessions + if str(row.get("user_id") or "") == principal["id"] + ] + else: + sessions = all_sessions + session_ids = {str(row["id"]) for row in sessions} + + all_memories = [dict(row) for row in conn.execute( + "SELECT * FROM memories WHERE workspace_id=? ORDER BY id", (wid,) + ).fetchall()] + if principal_scoped: + memories = [ + row for row in all_memories + if ( + str(row.get("scope") or "workspace") != "session" + or str(row.get("session_id") or "") in session_ids + ) + ] + else: + memories = all_memories + memory_ids = {str(row["id"]) for row in memories} + + all_entities = [dict(row) for row in conn.execute( + "SELECT * FROM entities WHERE workspace_id=? ORDER BY id", (wid,) + ).fetchall()] + workspace_entity_ids = {str(row["id"]) for row in all_entities} + all_edges = [dict(row) for row in conn.execute( + "SELECT * FROM edges WHERE workspace_id=? ORDER BY id", (wid,) + ).fetchall()] + all_supports = [dict(row) for row in conn.execute( + "SELECT support.* FROM edge_supports support " + "JOIN edges edge ON edge.id=support.edge_id " + "WHERE edge.workspace_id=? " + "ORDER BY support.edge_id, support.memory_id, support.source_kind, support.id", + (wid,), + ).fetchall()] + supports_by_edge: dict[str, list[dict]] = {} + for support in all_supports: + supports_by_edge.setdefault(str(support["edge_id"]), []).append(support) + + def _json_memory_references(raw: Any) -> set[str]: + try: + value = json.loads(raw or "{}") if isinstance(raw, str) else raw + except (TypeError, ValueError, RecursionError): + return set() + found: set[str] = set() + + def walk(item: Any) -> None: + if isinstance(item, dict): + for child in item.values(): + walk(child) + elif isinstance(item, (list, tuple)): + for child in item: + walk(child) + elif isinstance(item, str) and item.startswith("mem_"): + found.add(item) + + walk(value) + return found + + if principal_scoped: + edges = [] + for edge in all_edges: + if ( + str(edge.get("src") or "") not in workspace_entity_ids + or str(edge.get("dst") or "") not in workspace_entity_ids + ): + continue + edge_supports = supports_by_edge.get(str(edge["id"]), []) + visible_supports = [ + row for row in edge_supports + if str(row.get("memory_id") or "") in memory_ids + ] + provenance_refs = _json_memory_references(edge.get("provenance")) + if edge_supports and not visible_supports: + continue + if ( + not edge_supports + and provenance_refs + and provenance_refs.isdisjoint(memory_ids) + ): + continue + edges.append(edge) else: - memory_visibility = ( - " AND (COALESCE(m.scope, 'workspace')!='session' OR EXISTS (" - "SELECT 1 FROM sessions visible_session WHERE visible_session.id=m.session_id " - "AND visible_session.user_id=?))" + edges = all_edges + edge_ids = {str(row["id"]) for row in edges} + edge_supports = [ + row for row in all_supports + if ( + str(row.get("edge_id") or "") in edge_ids + and str(row.get("memory_id") or "") in memory_ids ) - session_visibility = " AND user_id=?" - visibility_params = [user["id"]] - memories = [dict(r) for r in conn.execute( - "SELECT m.* FROM memories m WHERE m.workspace_id=?" + memory_visibility - + " ORDER BY m.rowid", (wid, *visibility_params))] - sessions = [dict(r) for r in conn.execute( - "SELECT * FROM sessions WHERE workspace_id=?" + session_visibility - + " ORDER BY rowid", (wid, *visibility_params))] - audit = [dict(r) for r in conn.execute( - "SELECT a.* FROM audit a JOIN memories m ON m.id = a.target " - "WHERE m.workspace_id=?" + memory_visibility + " ORDER BY a.ts", - (wid, *visibility_params))] - receipts = self.store.list_receipts(workspace_id=wid, limit=10_000) - import time as _time - return {"format": "engraphis-export/1", "exported_at": _time.time(), - "workspace": workspace, "counts": {"memories": len(memories), - "sessions": len(sessions), "audit": len(audit), - "receipts": len(receipts)}, - "memories": memories, "sessions": sessions, "audit": audit, - "receipts": receipts} + ] + + all_incidence = [dict(row) for row in conn.execute( + "SELECT * FROM memory_entities WHERE workspace_id=? " + "ORDER BY memory_id, entity_id, source_kind, id", + (wid,), + ).fetchall()] + memory_entities = [ + row for row in all_incidence + if ( + str(row.get("memory_id") or "") in memory_ids + and str(row.get("entity_id") or "") in workspace_entity_ids + ) + ] + if principal_scoped: + entity_ids = { + str(edge["src"]) for edge in edges + } | { + str(edge["dst"]) for edge in edges + } | { + str(row["entity_id"]) for row in memory_entities + } + entities = [ + row for row in all_entities if str(row["id"]) in entity_ids + ] + else: + entities = all_entities + entity_ids = workspace_entity_ids + + # A malformed/cross-workspace endpoint is not portable workspace state. + edges = [ + row for row in edges + if str(row.get("src") or "") in entity_ids + and str(row.get("dst") or "") in entity_ids + ] + edge_ids = {str(row["id"]) for row in edges} + edge_supports = [ + row for row in edge_supports + if str(row.get("edge_id") or "") in edge_ids + ] + memory_entities = [ + row for row in memory_entities + if str(row.get("entity_id") or "") in entity_ids + ] + + memory_links = [dict(row) for row in conn.execute( + "SELECT link.* FROM mem_links link " + "JOIN memories left_memory ON left_memory.id=link.a " + "JOIN memories right_memory ON right_memory.id=link.b " + "WHERE left_memory.workspace_id=? AND right_memory.workspace_id=? " + "ORDER BY link.a, link.b, link.relation, link.layer, link.created_at", + (wid, wid), + ).fetchall()] + memory_links = [ + row for row in memory_links + if str(row.get("a") or "") in memory_ids + and str(row.get("b") or "") in memory_ids + ] + + if repo_ids: + symbols = [dict(row) for row in conn.execute( + "SELECT symbol.* FROM symbols symbol " + "JOIN repos repo ON repo.id=symbol.repo_id " + "WHERE repo.workspace_id=? ORDER BY symbol.id", + (wid,), + ).fetchall()] + code_edges = [dict(row) for row in conn.execute( + "SELECT edge.* FROM code_edges edge " + "JOIN repos repo ON repo.id=edge.repo_id " + "WHERE repo.workspace_id=? ORDER BY edge.id", + (wid,), + ).fetchall()] + code_files = [dict(row) for row in conn.execute( + "SELECT file.* FROM code_files file " + "JOIN repos repo ON repo.id=file.repo_id " + "WHERE repo.workspace_id=? ORDER BY file.repo_id, file.file", + (wid,), + ).fetchall()] + code_memory_links = [dict(row) for row in conn.execute( + "SELECT link.* FROM code_memory_links link " + "JOIN repos repo ON repo.id=link.repo_id " + "WHERE repo.workspace_id=? ORDER BY link.id", + (wid,), + ).fetchall()] + else: + symbols = [] + code_edges = [] + code_files = [] + code_memory_links = [] + symbol_ids = {str(row["id"]) for row in symbols} + code_memory_links = [ + row for row in code_memory_links + if ( + str(row.get("memory_id") or "") in memory_ids + and str(row.get("symbol_id") or "") in symbol_ids + ) + ] + + events = [dict(row) for row in conn.execute( + "SELECT * FROM events WHERE workspace_id=? ORDER BY id", (wid,) + ).fetchall()] + if principal_scoped: + events = [ + row for row in events + if ( + not row.get("session_id") + or str(row["session_id"]) in session_ids + ) + ] + event_ids = {str(row["id"]) for row in events} + + audit_by_id: dict[str, dict] = {} + for row in conn.execute( + "SELECT audit.* FROM audit audit " + "JOIN memories memory ON memory.id=audit.target " + "WHERE memory.workspace_id=? " + "UNION ALL SELECT audit.* FROM audit audit WHERE audit.target=?", + (wid, wid), + ).fetchall(): + item = dict(row) + if ( + (not principal_scoped and item["target"] == wid) + or str(item["target"]) in memory_ids + ): + audit_by_id[str(item["id"])] = item + audit = sorted( + audit_by_id.values(), + key=lambda row: ( + float(row.get("ts") or 0.0), + str(row.get("id") or ""), + ), + ) + audit_ids = {str(row["id"]) for row in audit} + + receipts = self._complete_receipt_rows(wid) + receipt_ids = {str(row.get("id") or "") for row in receipts} + receipt_chain_row = conn.execute( + "SELECT receipt_count, head_hash, integrity_error, updated_at " + "FROM receipt_chain_heads WHERE workspace_id=?", + (wid,), + ).fetchone() + receipt_chain = dict(receipt_chain_row) if receipt_chain_row is not None else None + if receipt_chain is not None: + raw_count = receipt_chain.get("receipt_count") + if type(raw_count) is not int or raw_count < 0: + receipt_chain["receipt_count"] = None + raw_updated_at = receipt_chain.get("updated_at") + if ( + type(raw_updated_at) not in (int, float) + or not math.isfinite(float(raw_updated_at)) + ): + receipt_chain["updated_at"] = None + raw_error = str(receipt_chain.get("integrity_error") or "") + if raw_error not in { + "", "pre_append_anchor_mismatch", "pre_append_anchor_missing", + "pre_append_chain_corruption", "migration_chain_invalid", + }: + receipt_chain["integrity_error"] = self._redacted_receipt_value( + raw_error + ) + receipt_chain["head_hash"] = self._safe_receipt_hash( + receipt_chain.get("head_hash"), allow_empty=True + ) + receipt_verification = self._safe_receipt_verification( + self.store.verify_receipts(workspace_id=wid) + ) + + # Scrub structured references that point outside the export boundary. This is + # essential for Team exports: dropping a private memory while leaving its id in + # provenance, incidence, or event refs is still a privacy leak. + allowed_reference_ids = { + wid, + *repo_ids, + *session_ids, + *memory_ids, + *entity_ids, + *edge_ids, + *symbol_ids, + *event_ids, + *audit_ids, + *receipt_ids, + *(str(row.get("id") or "") for row in edge_supports), + *(str(row.get("id") or "") for row in memory_entities), + *(str(row.get("id") or "") for row in code_edges), + *(str(row.get("id") or "") for row in code_memory_links), + } + typed_reference = re.compile( + r"^(?:ws|repo|ses|mem|ent|edg|sym|evt|job|aud|dev|rcpt)_[A-Za-z0-9_-]+$" + ) + embedded_reference = re.compile( + r"(?:ws|repo|ses|mem|ent|edg|sym|evt|job|aud|dev|rcpt)_[A-Za-z0-9_-]+" + ) + dropped = object() + + def scrub_value(value: Any) -> Any: + if isinstance(value, dict): + clean: dict = {} + for key, child in value.items(): + scrubbed = scrub_value(child) + if scrubbed is not dropped: + clean[key] = scrubbed + return clean + if isinstance(value, list): + return [ + scrubbed for child in value + if (scrubbed := scrub_value(child)) is not dropped + ] + if isinstance(value, tuple): + return scrub_value(list(value)) + if isinstance(value, str): + if typed_reference.fullmatch(value) and value not in allowed_reference_ids: + return dropped + return embedded_reference.sub( + lambda match: ( + "[redacted]" + if match.group(0) not in allowed_reference_ids + else match.group(0) + ), + value, + ) + return value + + def scrub_json(raw: Any, default: Any) -> Any: + if not isinstance(raw, str): + return raw + try: + value = json.loads(raw) + except (TypeError, ValueError, RecursionError): + return embedded_reference.sub( + lambda match: ( + "[redacted]" + if match.group(0) not in allowed_reference_ids + else match.group(0) + ), + raw, + ) + clean = scrub_value(value) + if clean is dropped: + clean = default + if not canonical and clean == value: + return raw + return json.dumps( + clean, + sort_keys=canonical, + separators=(",", ":") if canonical else None, + ensure_ascii=False, + ) + + workspace_row["settings"] = scrub_json(workspace_row.get("settings"), {}) + if principal_scoped: + try: + public_settings = json.loads(workspace_row.get("settings") or "{}") + except (TypeError, ValueError, RecursionError): + public_settings = {} + if isinstance(public_settings, dict): + public_settings.pop("owner", None) + workspace_row["settings"] = json.dumps( + public_settings, + sort_keys=canonical, + separators=(",", ":") if canonical else None, + ensure_ascii=False, + ) + # Do not expose server-local placement or credential-bearing remote URLs to a + # remote Team member. Code/file records remain fully portable. + for repo in repos: + repo["root_path"] = None + repo["vcs_remote"] = None + for repo in repos: + repo["settings"] = scrub_json(repo.get("settings"), {}) + for session in sessions: + session["open_threads"] = scrub_json(session.get("open_threads"), []) + for memory in memories: + if str(memory.get("session_id") or "") not in session_ids: + memory["session_id"] = None + memory["keywords"] = scrub_json(memory.get("keywords"), []) + memory["metadata"] = scrub_json(memory.get("metadata"), {}) + memory["provenance"] = scrub_json(memory.get("provenance"), {}) + for entity in entities: + if str(entity.get("canonical_id") or "") not in entity_ids: + entity["canonical_id"] = entity["id"] + for edge in edges: + edge["provenance"] = scrub_json(edge.get("provenance"), {}) + for support in edge_supports: + support["provenance"] = scrub_json(support.get("provenance"), {}) + for incidence in memory_entities: + incidence["provenance"] = scrub_json(incidence.get("provenance"), {}) + for event in events: + event["refs"] = scrub_json(event.get("refs"), []) + for item in audit: + item["detail"] = embedded_reference.sub( + lambda match: ( + "[redacted]" + if match.group(0) not in allowed_reference_ids + else match.group(0) + ), + str(item.get("detail") or ""), + ) + + table_rows = { + "repos": repos, + "sessions": sessions, + "memories": memories, + "entities": entities, + "edges": edges, + "edge_supports": edge_supports, + "memory_entities": memory_entities, + "memory_links": memory_links, + "symbols": symbols, + "code_edges": code_edges, + "code_files": code_files, + "code_memory_links": code_memory_links, + "events": events, + "audit": audit, + "receipts": receipts, + } + payload = { + "format": "engraphis-export/2", + "workspace": workspace, + "workspace_record": workspace_row, + "schema_version": self.store.schema_version, + "visibility": "principal" if principal_scoped else "workspace", + "counts": { + name: len(rows) for name, rows in table_rows.items() + }, + "ordering": { + "repos": ["id"], + "sessions": ["id"], + "memories": ["id"], + "entities": ["id"], + "edges": ["id"], + "edge_supports": [ + "edge_id", "memory_id", "source_kind", "id", + ], + "memory_entities": [ + "memory_id", "entity_id", "source_kind", "id", + ], + "memory_links": [ + "a", "b", "relation", "layer", "created_at", + ], + "symbols": ["id"], + "code_edges": ["id"], + "code_files": ["repo_id", "file"], + "code_memory_links": ["id"], + "events": ["id"], + "audit": ["ts", "id"], + "receipts": [ + "verified predecessor order; deterministic digest order if corrupt", + ], + }, + "completeness": { + "durable_workspace_state": True, + "receipts": True, + "omitted_nonportable_or_regenerable_tables": [ + "mem_fts", + "mem_vectors", + "mem_vec_ann", + "jobs", + "graph_index_state", + ], + }, + **table_rows, + "receipt_chain": receipt_chain, + "receipt_verification": receipt_verification, + } + if canonical: + # The table queries and filters above already use their declared stable + # ordering. The digest intentionally excludes wall-clock export time. + payload["canonical"] = True + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ).encode("utf-8") + return { + **payload, + "sha256": hashlib.sha256(encoded).hexdigest(), + } + payload["exported_at"] = time.time() + return payload def _recover_stale_graph_jobs(self, workspace_id: Optional[str] = None) -> int: """Fail expired process-local workers and release their rebuilding gate. @@ -3383,7 +4366,9 @@ def _graph_index_info(self, workspace_id: str) -> dict: ).fetchone() if row is None: return { - "generation": self.store.schema_version, + # Graph generations are write-trigger revisions, independent + # of the database schema version. + "generation": 0, "state": "ready", "active_job_id": None, "updated_at": None, @@ -3904,6 +4889,8 @@ def _run_graph_index_job(self, job_id: str) -> None: def _graph_scene_rows(self, *, workspace: str, repo: Optional[str] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, entity_types: Optional[list[str]] = None, memory_types: Optional[list[str]] = None, time_from: Optional[float] = None, @@ -3924,6 +4911,8 @@ def _graph_scene_rows(self, *, workspace: str, repo: Optional[str] = None, workspace=clean_workspace, repo=repo, as_of=as_of, + valid_at=valid_at, + known_at=known_at, entity_types=entity_types, memory_types=memory_types, time_from=time_from, @@ -3933,7 +4922,7 @@ def _graph_scene_rows(self, *, workspace: str, repo: Optional[str] = None, include_complete_rows=include_complete_rows, ) index_info = self._graph_index_info(rows[1]) if rows[1] else { - "generation": self.store.schema_version, + "generation": 0, "state": "ready", "active_job_id": None, "updated_at": None, @@ -3949,6 +4938,8 @@ def _graph_scene_rows(self, *, workspace: str, repo: Optional[str] = None, def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, entity_types: Optional[list[str]] = None, memory_types: Optional[list[str]] = None, time_from: Optional[float] = None, @@ -3986,12 +4977,12 @@ def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = No repo_id = self._lookup_repo(wid, repo_name) if repo_id is None: raise ValidationError(f"no repo named '{repo_name}' in workspace '{ws}'") - try: - t = float(as_of) if as_of is not None else time.time() - except (TypeError, ValueError, OverflowError): - raise ValidationError("as_of must be a finite timestamp") - if not math.isfinite(t): - raise ValidationError("as_of must be a finite timestamp") + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) + present = time.time() + t = valid_at if valid_at is not None else present + known_t = known_at if known_at is not None else present try: lower_time = float(time_from) if time_from is not None else None upper_time = float(time_to) if time_to is not None else None @@ -4007,8 +4998,9 @@ def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = No "normalized_name, canonical_method, canonical_confidence, created_at " "FROM entities entity WHERE workspace_id=? AND " + _graph_entity_visibility_sql("entity", at=t) + + " AND (created_at IS NULL OR created_at<=?)" ) - entity_params: list[Any] = [wid] + entity_params: list[Any] = [wid, known_t] if repo_id: entity_sql += " AND (repo_id=? OR repo_id IS NULL)" entity_params.append(repo_id) @@ -4037,9 +5029,12 @@ def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = No "SELECT id, workspace_id, repo_id, src, dst, relation, layer, weight, " "valid_from, valid_to, ingested_at, expired_at, provenance FROM edges " "WHERE workspace_id=? AND (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? remaining_entities: if include_complete_rows: @@ -4151,8 +5203,16 @@ def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = No remaining_edges = MAX_GRAPH_ANALYSIS_EDGES - len(edge_rows) code_edges = self.store.conn.execute( "SELECT id, src, dst, relation, layer FROM code_edges " - "WHERE repo_id=? ORDER BY id LIMIT ?", - (repo_row["id"], remaining_edges + 1), + "WHERE repo_id=? AND (valid_from IS NULL OR valid_from<=?) " + "AND (valid_to IS NULL OR ? remaining_edges: if include_complete_rows: @@ -4201,8 +5261,12 @@ def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = No # Bounded IN chunks avoid a second scan of the relation table while preserving # the exact selected edge ids. Weak co-occurrence is filtered after canonical # relation bundling, once its aggregate support is known. + scene_filter = SearchFilter( + workspace_id=wid, repo_id=repo_id, include_ancestors=True, + valid_at=t, known_at=known_t, + ) support_rows = self.store.edge_supports_in_scope( - edge_ids, at=t, limit=MAX_GRAPH_ANALYSIS_SUPPORTS + 1 + edge_ids, flt=scene_filter, limit=MAX_GRAPH_ANALYSIS_SUPPORTS + 1 ) if len(support_rows) > MAX_GRAPH_ANALYSIS_SUPPORTS: if include_complete_rows: @@ -4228,9 +5292,14 @@ def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = No "SELECT id, mtype, COALESCE(valid_from, ingested_at, 0) AS support_time " "FROM memories WHERE workspace_id=? AND id IN (" + marks + ") " "AND (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? MAX_GRAPH_COMPLETE_MEMORY_LINKS: raise GraphSceneCapacityExceeded( @@ -4319,9 +5400,17 @@ def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = No "SELECT links.id, links.repo_id, links.symbol_id, links.memory_id, " "links.relation, links.confidence FROM code_memory_links links " "JOIN selected_memory memory ON memory.id=links.memory_id " - "JOIN repos repo ON repo.id=links.repo_id WHERE repo.workspace_id=?" + "JOIN repos repo ON repo.id=links.repo_id WHERE repo.workspace_id=? " + "AND (links.valid_from IS NULL OR links.valid_from<=?) " + "AND (links.valid_to IS NULL OR ? int: raise ValidationError("min_confidence must be a finite number") if not math.isfinite(clean_min_confidence) or not 0.0 <= clean_min_confidence <= 1.0: raise ValidationError("min_confidence must be between 0 and 1") - try: - clean_as_of = float(as_of) if as_of is not None else None - except (TypeError, ValueError, OverflowError): - raise ValidationError("as_of must be a finite timestamp") - if clean_as_of is not None and not math.isfinite(clean_as_of): - raise ValidationError("as_of must be a finite timestamp") + clean_as_of, clean_valid_at, clean_known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) try: clean_time_from = float(time_from) if time_from is not None else None clean_time_to = float(time_to) if time_to is not None else None @@ -4460,13 +5549,16 @@ def bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int: revision, clean_workspace, clean_level, clean_center_id or "", clean_system_id or "", tuple(clean_seeds), clean_repo or "", tuple(clean_layers or ()), tuple(clean_relations), tuple(clean_entity_types), - tuple(clean_memory_types), clean_as_of, clean_time_from, clean_time_to, + tuple(clean_memory_types), clean_as_of, clean_valid_at, clean_known_at, + clean_time_from, clean_time_to, clean_depth, clean_min_support, clean_min_confidence, bool(include_weak_cooccurrence), bool(include_code), clean_node_limit, clean_edge_limit, ) cached = self._graph_scene_cache.get(cache_key) - if cached is not None and (clean_as_of is not None or time.time() < cached[0]): + if cached is not None and ( + clean_valid_at is not None or clean_known_at is not None + or time.time() < cached[0]): self._graph_scene_cache.move_to_end(cache_key) scene = copy.deepcopy(cached[1]) scene["meta"]["cache_hit"] = True @@ -4476,10 +5568,13 @@ def bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int: return scene if cached is not None: del self._graph_scene_cache[cache_key] - query_at = clean_as_of if clean_as_of is not None else time.time() + present = time.time() + query_at = clean_valid_at if clean_valid_at is not None else present + query_known_at = clean_known_at if clean_known_at is not None else present (ws, _wid, entities, edges, supports, memories, memory_links, code_memory_links, index_info) = self._graph_scene_rows( - workspace=clean_workspace, repo=clean_repo, as_of=query_at, + workspace=clean_workspace, repo=clean_repo, + valid_at=query_at, known_at=query_known_at, entity_types=clean_entity_types, memory_types=clean_memory_types, time_from=clean_time_from, time_to=clean_time_to, include_weak_cooccurrence=include_weak_cooccurrence, @@ -4495,6 +5590,8 @@ def bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int: "entity_types": clean_entity_types, "memory_types": clean_memory_types, "as_of": clean_as_of, + "valid_at": clean_valid_at, + "known_at": clean_known_at, "time_from": clean_time_from, "time_to": clean_time_to, "min_support": clean_min_support, @@ -4539,7 +5636,9 @@ def bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int: ) scene["meta"]["payload_bytes_estimate"] = payload_bytes valid_until = ( - math.inf if clean_as_of is not None or not _wid + math.inf if ( + clean_valid_at is not None or clean_known_at is not None or not _wid + ) else self._graph_scene_valid_until(_wid, query_at) ) # One complete scene can be many megabytes. Keep at most one in the shared @@ -4557,6 +5656,8 @@ def graph_suggest(self, query: str, *, workspace: str, limit: int = 8, repo: Optional[str] = None, memory_types: Optional[list[str]] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, time_from: Optional[float] = None, time_to: Optional[float] = None, include_weak_cooccurrence: bool = False) -> dict: @@ -4580,14 +5681,18 @@ def graph_suggest(self, query: str, *, workspace: str, limit: int = 8, repo_id = self._lookup_repo(wid, clean_repo) if repo_id is None: raise ValidationError(f"no repo named '{clean_repo}' in workspace '{ws}'") + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) + present = time.time() + suggestion_at = valid_at if valid_at is not None else present + suggestion_known_at = known_at if known_at is not None else present try: - suggestion_at = float(as_of) if as_of is not None else time.time() lower_time = float(time_from) if time_from is not None else None upper_time = float(time_to) if time_to is not None else None except (TypeError, ValueError, OverflowError): raise ValidationError("graph suggestion times must be finite timestamps") - if (not math.isfinite(suggestion_at) - or (lower_time is not None and not math.isfinite(lower_time)) + if ((lower_time is not None and not math.isfinite(lower_time)) or (upper_time is not None and not math.isfinite(upper_time))): raise ValidationError("graph suggestion times must be finite timestamps") if lower_time is not None and upper_time is not None and lower_time > upper_time: @@ -4603,6 +5708,21 @@ def graph_suggest(self, query: str, *, workspace: str, limit: int = 8, like = f"%{escaped}%" prefix = f"{escaped}%" + def visible_sql(alias: str) -> tuple[str, list[float]]: + prefix = f"{alias}." + return ( + f"({prefix}valid_from IS NULL OR {prefix}valid_from<=?) " + f"AND ({prefix}valid_to IS NULL OR ?<{prefix}valid_to " + f"OR ({prefix}valid_to_recorded_at IS NOT NULL " + f"AND ?<{prefix}valid_to_recorded_at)) " + f"AND ({prefix}ingested_at IS NULL OR {prefix}ingested_at<=?) " + f"AND ({prefix}expired_at IS NULL OR ?<{prefix}expired_at)", + [ + suggestion_at, suggestion_at, suggestion_known_at, + suggestion_known_at, suggestion_known_at, + ], + ) + # Search identity rows directly instead of rebuilding Louvain/PageRank for each # keystroke. A canonical entity id also resolves to its current deterministic # community in ``build_graph_scene``, so the same stable result can represent an @@ -4612,8 +5732,11 @@ def graph_suggest(self, query: str, *, workspace: str, limit: int = 8, "FROM entities entity WHERE workspace_id=? AND (" "normalized_name LIKE ? ESCAPE '\\' OR canonical_id=? OR id=?) AND " + _graph_entity_visibility_sql("entity", at=suggestion_at) + + " AND (created_at IS NULL OR created_at<=?)" ) - entity_params: list[Any] = [wid, like, clean_query, clean_query] + entity_params: list[Any] = [ + wid, like, clean_query, clean_query, suggestion_known_at, + ] if repo_id: entity_sql += " AND (repo_id=? OR repo_id IS NULL)" entity_params.append(repo_id) @@ -4685,8 +5808,11 @@ def useful_identity(item: tuple[str, list[dict]]) -> bool: f"FROM entities entity WHERE workspace_id=? " f"AND canonical_id IN ({marks}) AND " + _graph_entity_visibility_sql("entity", at=suggestion_at) + + " AND (created_at IS NULL OR created_at<=?)" ) - member_params: list[Any] = [wid, *selected_canonical_ids] + member_params: list[Any] = [ + wid, *selected_canonical_ids, suggestion_known_at, + ] if repo_id: member_sql += " AND (repo_id=? OR repo_id IS NULL)" member_params.append(repo_id) @@ -4708,28 +5834,40 @@ def useful_identity(item: tuple[str, list[dict]]) -> bool: for start in range(0, len(member_ids), 400): chunk = member_ids[start:start + 400] marks = ",".join("?" for _ in chunk) + relation_visibility, relation_visibility_params = visible_sql("relation") + support_visibility, support_visibility_params = visible_sql("support") + memory_visibility, memory_visibility_params = visible_sql("memory") + temporal_visibility = ( + f"AND {relation_visibility} AND {support_visibility} " + f"AND {memory_visibility} " + ) support_sql = ( "SELECT endpoint, memory_id FROM (" "SELECT relation.src AS endpoint, support.memory_id FROM edges relation " "JOIN edge_supports support ON support.edge_id=relation.id " "JOIN memories memory ON memory.id=support.memory_id " f"WHERE relation.workspace_id=? AND relation.src IN ({marks}) " - "AND relation.valid_to IS NULL AND relation.expired_at IS NULL " - "AND support.valid_to IS NULL AND support.expired_at IS NULL " - "AND memory.valid_to IS NULL AND memory.expired_at IS NULL " + + temporal_visibility + "AND COALESCE(memory.scope, 'workspace')!='session' " "UNION ALL " "SELECT relation.dst AS endpoint, support.memory_id FROM edges relation " "JOIN edge_supports support ON support.edge_id=relation.id " "JOIN memories memory ON memory.id=support.memory_id " f"WHERE relation.workspace_id=? AND relation.dst IN ({marks}) " - "AND relation.valid_to IS NULL AND relation.expired_at IS NULL " - "AND support.valid_to IS NULL AND support.expired_at IS NULL " - "AND memory.valid_to IS NULL AND memory.expired_at IS NULL " + + temporal_visibility + "AND COALESCE(memory.scope, 'workspace')!='session')" ) + temporal_params = [ + *relation_visibility_params, + *support_visibility_params, + *memory_visibility_params, + ] rows = self.store.conn.execute( - support_sql, (wid, *chunk, wid, *chunk) + support_sql, + ( + wid, *chunk, *temporal_params, + wid, *chunk, *temporal_params, + ), ).fetchall() for row in rows: canonical_id = member_to_canonical.get(str(row["endpoint"]), "") @@ -4780,14 +5918,16 @@ def useful_identity(item: tuple[str, list[dict]]) -> bool: relations_out = [] code_symbols = [] if wid: + memory_visibility, memory_visibility_params = visible_sql("memories") memory_sql = ( "SELECT id, title, content, mtype, repo_id FROM memories " - "WHERE workspace_id=? AND (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? bool: relation_sql = ( "SELECT relation, COUNT(*) AS count FROM edges relation_edge " "WHERE workspace_id=? " - "AND relation LIKE ? ESCAPE '\\' AND (valid_from IS NULL OR valid_from<=?) " - "AND (valid_to IS NULL OR ? bool: "OR lower(s.fqname) LIKE ? ESCAPE '\\')" ) symbol_params: list[Any] = [wid, like, like] + symbol_visibility, symbol_visibility_params = visible_sql("s") + symbol_sql += f" AND {symbol_visibility}" + symbol_params.extend(symbol_visibility_params) if repo_id: symbol_sql += " AND s.repo_id=?" symbol_params.append(repo_id) @@ -4855,6 +6015,7 @@ def useful_identity(item: tuple[str, list[dict]]) -> bool: } for row in symbol_rows] return { "workspace": ws, "query": clean_query, + "as_of": as_of, "valid_at": valid_at, "known_at": known_at, "groups": { "systems": system_results, "entities": entity_results, "memories": memories, "repositories": repositories, @@ -4866,15 +6027,22 @@ def graph_entity(self, canonical_id: str, *, workspace: str, repo: Optional[str] = None, memory_types: Optional[list[str]] = None, as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None, time_from: Optional[float] = None, time_to: Optional[float] = None, include_weak_cooccurrence: bool = True) -> dict: clean_canonical_id = _clean_text( canonical_id, field="canonical_id", max_chars=MAX_NAME_CHARS ) + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) + history_known_at = known_at if known_at is not None else time.time() (ws, wid, entities, edges, supports, _memories, _memory_links, _code_memory_links, _index_info) = self._graph_scene_rows( workspace=workspace, repo=repo, as_of=as_of, + valid_at=valid_at, known_at=known_at, memory_types=memory_types, time_from=time_from, time_to=time_to, include_weak_cooccurrence=include_weak_cooccurrence, ) @@ -4937,8 +6105,9 @@ def graph_entity(self, canonical_id: str, *, workspace: str, chunk = ordered_ids[start:start + 500] marks = ",".join("?" for _ in chunk) for memory in self.store.conn.execute( - "SELECT id, title, content, mtype, valid_from, valid_to, ingested_at, " - "expired_at, provenance FROM memories WHERE workspace_id=? " + "SELECT id, title, content, mtype, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at, provenance " + "FROM memories WHERE workspace_id=? " "AND COALESCE(scope, 'workspace')!='session' " "AND id IN (" + marks + ") " "ORDER BY id", (wid, *chunk) @@ -4957,6 +6126,7 @@ def graph_entity(self, canonical_id: str, *, workspace: str, "source_kind": support.get("source_kind", "legacy_unknown"), "confidence": float(support.get("confidence", 0.5)), "valid_from": memory["valid_from"], "valid_to": memory["valid_to"], + "valid_to_recorded_at": memory["valid_to_recorded_at"], "ingested_at": memory["ingested_at"], "expired_at": memory["expired_at"], "provenance": memory_provenance, }) @@ -4967,11 +6137,17 @@ def graph_entity(self, canonical_id: str, *, workspace: str, )) member_ids = node["member_ids"] history_filter = ( - "workspace_id=? AND (valid_to IS NOT NULL OR expired_at IS NOT NULL) " + "workspace_id=? AND (ingested_at IS NULL OR ingested_at<=?) " + "AND ((valid_to IS NOT NULL AND " + "(valid_to_recorded_at IS NULL OR valid_to_recorded_at<=?)) " + "OR (expired_at IS NOT NULL AND expired_at<=?)) " "AND (src IN (SELECT id FROM entities WHERE workspace_id=? AND canonical_id=?) " "OR dst IN (SELECT id FROM entities WHERE workspace_id=? AND canonical_id=?))" ) - history_params: tuple[Any, ...] = (wid, wid, resolved, wid, resolved) + history_params: tuple[Any, ...] = ( + wid, history_known_at, history_known_at, history_known_at, + wid, resolved, wid, resolved, + ) history_filter += ( " AND (NOT EXISTS (SELECT 1 FROM edge_supports any_history_support " "WHERE any_history_support.edge_id=edges.id) OR EXISTS (" @@ -4988,7 +6164,8 @@ def graph_entity(self, canonical_id: str, *, workspace: str, ).fetchone()["n"]) history = [dict(row) for row in self.store.conn.execute( "SELECT id, src, dst, relation, layer, weight, valid_from, valid_to, " - "ingested_at, expired_at FROM edges WHERE " + history_filter + " " + "valid_to_recorded_at, ingested_at, expired_at FROM edges WHERE " + + history_filter + " " "ORDER BY COALESCE(valid_to, expired_at, valid_from, ingested_at) DESC, id DESC " "LIMIT ?", (*history_params, GRAPH_ENTITY_HISTORY_LIMIT), @@ -5020,10 +6197,14 @@ def graph_entity(self, canonical_id: str, *, workspace: str, "history": history_total > len(history), }, "as_of": as_of, + "valid_at": valid_at, + "known_at": known_at, } def graph_entity_evidence(self, canonical_id: str, *, workspace: str, - as_of: Optional[float] = None) -> dict: + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> dict: """Return one graph entity's public supporting memories without rebuilding the graph. The full entity inspector calculates canonical relations, history, and graph metrics, @@ -5038,12 +6219,12 @@ def graph_entity_evidence(self, canonical_id: str, *, workspace: str, if wid is None: raise ValidationError(f"no workspace '{ws}'") self._assert_graph_index_ready(wid) - try: - anchor = float(as_of) if as_of is not None else time.time() - except (TypeError, ValueError, OverflowError) as exc: - raise ValidationError("as_of must be a finite timestamp") from exc - if not math.isfinite(anchor): - raise ValidationError("as_of must be a finite timestamp") + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) + present = time.time() + anchor = valid_at if valid_at is not None else present + known_anchor = known_at if known_at is not None else present target = self.store.conn.execute( "SELECT id, canonical_id FROM entities WHERE workspace_id=? AND id=? LIMIT 1", @@ -5065,19 +6246,35 @@ def graph_entity_evidence(self, canonical_id: str, *, workspace: str, support_conditions = ( "relation.workspace_id=? AND relation.{endpoint}=target.id " "AND (relation.valid_from IS NULL OR relation.valid_from<=?) " - "AND (relation.valid_to IS NULL OR ? dict: + as_of: Optional[float] = None, + valid_at: Optional[float] = None, + known_at: Optional[float] = None) -> dict: """Entity-relation network for a workspace: nodes/edges plus type counts, top-connected entities, and connectivity stats — powers the Graph tab in both the v1-look dashboard and the Inspector UI (engraphis.graphdata @@ -5207,6 +6413,9 @@ def graph(self, *, workspace: str, limit: int = 2000, original dashboard-only implementation, which read the DB file directly and skipped this check entirely.""" ws = self._clean_ws(workspace) # binding enforced here, before any lookup + as_of, valid_at, known_at = _temporal_anchors( + as_of=as_of, valid_at=valid_at, known_at=known_at + ) wid = self._lookup_workspace(ws) if wid is None: return empty_graph(ws) @@ -5219,80 +6428,114 @@ def graph(self, *, workspace: str, limit: int = 2000, limit = max(1, min(node_limit, int(limit))) conn = self.store.conn restrict_sessions = True - temporal_anchor = None - if as_of is not None: - try: - temporal_anchor = float(as_of) - except (TypeError, ValueError) as exc: - raise ValidationError("as_of must be a timestamp") from exc - if not math.isfinite(temporal_anchor): - raise ValidationError("as_of must be a finite timestamp") - # Code symbols and their edges are an index of the checkout as it exists now; - # unlike memory and entity relations, they do not carry world-time validity. - # Do not blend that live overlay into an otherwise historical graph response. - include_live_code = include_code and temporal_anchor is None + present = time.time() + world_anchor = valid_at if valid_at is not None else present + system_anchor = known_at if known_at is not None else present + # ``as_of``/``valid_at`` powers the Time view, which intentionally retains + # superseded public relations as ghosts. A system-time-only read remains an + # exact world-time snapshot while limiting the graph to what was then known. + include_relation_history = valid_at is not None + temporal_requested = valid_at is not None or known_at is not None + + def temporal_sql(alias: str, *, history: bool = False + ) -> tuple[str, list[float]]: + """Parameterized world/system visibility for graph-owned SQL. + + ``history`` retains world-time closures for the Time view, but never + relaxes system time: future ingestion and later expiry stay invisible. + """ + prefix = f"{alias}." + if history: + world_sql = f"({prefix}valid_from IS NULL OR {prefix}valid_from<=?)" + params: list[float] = [world_anchor] + else: + world_sql = ( + f"({prefix}valid_from IS NULL OR {prefix}valid_from<=?) " + f"AND ({prefix}valid_to IS NULL OR ?<{prefix}valid_to " + f"OR ({prefix}valid_to_recorded_at IS NOT NULL " + f"AND ?<{prefix}valid_to_recorded_at))" + ) + params = [world_anchor, world_anchor, system_anchor] + return ( + world_sql + + f" AND ({prefix}ingested_at IS NULL OR {prefix}ingested_at<=?)" + + f" AND ({prefix}expired_at IS NULL OR ?<{prefix}expired_at)", + [*params, system_anchor, system_anchor], + ) + + def public_edge_sql(alias: str, *, history: bool = False + ) -> tuple[str, list[float]]: + """Evidence visibility for one edge alias, including session isolation.""" + support_sql, support_params = temporal_sql( + "visibility_support", history=history + ) + memory_sql, memory_params = temporal_sql( + "visibility_memory", history=history + ) + return ( + "(NOT EXISTS (SELECT 1 FROM edge_supports any_support " + f"WHERE any_support.edge_id={alias}.id) OR EXISTS (" + "SELECT 1 FROM edge_supports visibility_support " + "JOIN memories visibility_memory " + "ON visibility_memory.id=visibility_support.memory_id " + f"WHERE visibility_support.edge_id={alias}.id " + f"AND {support_sql} AND {memory_sql} " + f"AND visibility_memory.workspace_id={alias}.workspace_id " + "AND COALESCE(visibility_memory.scope, 'workspace')!='session'))", + [*support_params, *memory_params], + ) def visible_entities(): - """Return public graph entities without a correlated edge scan per node. - - The older predicate called ``_graph_entity_visibility_sql`` directly - for every candidate entity. On a mature local workspace that became - thousands of nested ``edges``/``edge_supports`` probes before the - renderer even received a response. Aggregate an edge's visibility - once, then aggregate that result by endpoint: no-edge entities remain - visible, entities supported only by session memories remain hidden, - and mixed-history entities stay visible as before. + """Return public entities under both temporal anchors in one bounded query. + + An entity with no relation history remains a public/manual node. Once an + entity has relation history, however, it is visible only when at least one + touching relation has public evidence at the selected anchors. This avoids + revealing session-only or future-supported identities. """ - evidence_at = "" - if temporal_anchor is not None: - # Entity records are backfilled lazily, so their ``created_at`` is - # ingestion time rather than world time. Anchor the evidence instead: - # a later public support must not reveal an entity that was private at - # the requested point in time. Do not bound ``valid_to`` here; the - # Time view deliberately retains previously-public facts as ghosts. - anchor = repr(temporal_anchor) - evidence_at = ( - " AND (support.valid_from IS NULL " - f"OR support.valid_from<={anchor}) " - "AND (visibility_memory.valid_from IS NULL " - f"OR visibility_memory.valid_from<={anchor})" - ) + relation_sql, relation_params = temporal_sql( + "relation", history=include_relation_history + ) + public_sql, public_params = public_edge_sql( + "relation", history=include_relation_history + ) sql = f""" WITH edge_visibility AS ( - SELECT relation.id, relation.src, relation.dst, - MAX(CASE - WHEN support.edge_id IS NULL THEN 1 - WHEN visibility_memory.id IS NOT NULL - AND COALESCE(visibility_memory.scope, 'workspace') != 'session' - {evidence_at} - THEN 1 ELSE 0 - END) AS is_visible + SELECT relation.id, relation.src, relation.dst FROM edges relation - LEFT JOIN edge_supports support ON support.edge_id=relation.id - LEFT JOIN memories visibility_memory - ON visibility_memory.id=support.memory_id - WHERE relation.workspace_id=? - GROUP BY relation.id - ), endpoint_visibility AS ( - SELECT src AS entity_id, is_visible FROM edge_visibility + WHERE relation.workspace_id=? AND {relation_sql} + AND {public_sql} + ), all_endpoint AS ( + SELECT src AS entity_id FROM edges WHERE workspace_id=? UNION ALL - SELECT dst AS entity_id, is_visible FROM edge_visibility + SELECT dst AS entity_id FROM edges WHERE workspace_id=? + ), entity_history AS ( + SELECT entity_id, COUNT(*) AS degree + FROM all_endpoint GROUP BY entity_id + ), visible_endpoint AS ( + SELECT src AS entity_id FROM edge_visibility + UNION ALL + SELECT dst AS entity_id FROM edge_visibility ), entity_visibility AS ( - SELECT entity_id, MAX(is_visible) AS is_visible, - COUNT(*) AS degree - FROM endpoint_visibility - GROUP BY entity_id + SELECT entity_id, COUNT(*) AS degree + FROM visible_endpoint GROUP BY entity_id ) SELECT entity.id, entity.name, entity.etype, repo.name AS repo, entity.created_at AS valid_from, COUNT(*) OVER() AS visible_total FROM entities entity LEFT JOIN repos repo ON repo.id=entity.repo_id + LEFT JOIN entity_history history ON history.entity_id=entity.id LEFT JOIN entity_visibility visible ON visible.entity_id=entity.id WHERE entity.workspace_id=? - AND COALESCE(visible.is_visible, 1)=1 + AND (entity.created_at IS NULL OR entity.created_at<=?) + AND (COALESCE(history.degree, 0)=0 + OR COALESCE(visible.degree, 0)>0) """ - params: list[Any] = [wid, wid] + params: list[Any] = [ + wid, *relation_params, *public_params, wid, wid, + wid, system_anchor, + ] if connected_only: sql += " AND COALESCE(visible.degree, 0)>0" sql += " ORDER BY COALESCE(visible.degree, 0) DESC, entity.id LIMIT ?" @@ -5304,8 +6547,14 @@ def visible_entities(): # structured-metadata graph bridge. On first Graph-tab open in a process, feed # the missing graph state once; feed() de-dupes entities/edges. # Strictly read-only surfaces disable this write-on-first-read migration. - if backfill and self._should_backfill_graph(wid, bool(ents)): + if (backfill and not temporal_requested + and self._should_backfill_graph(wid, bool(ents))): self._lazy_backfill_graph(wid) + # Rows created by the migration must be part of the same current read. + # Explicit historical anchors never enter this write path. + present = time.time() + world_anchor = present + system_anchor = present ents = visible_entities() visible_total = int(ents[0]["visible_total"]) if ents else 0 if full and visible_total > MAX_GRAPH_ANALYSIS_ENTITIES: @@ -5335,7 +6584,8 @@ def visible_entities(): # here as a useful fallback for both Graph-tab clients. memory_link_fallback: list[dict] = [] if not entity_rows and selected_graph_layers != []: - now = temporal_anchor if temporal_anchor is not None else time.time() + left_visibility, left_params = temporal_sql("left_memory") + right_visibility, right_params = temporal_sql("right_memory") sql = ( "SELECT link.a, link.b, link.relation, " "COALESCE(link.layer, 'semantic') AS layer, " @@ -5352,14 +6602,18 @@ def visible_entities(): "WHERE left_memory.workspace_id=? AND right_memory.workspace_id=? " "AND COALESCE(left_memory.scope, 'workspace')!='session' " "AND COALESCE(right_memory.scope, 'workspace')!='session' " - "AND (left_memory.valid_from IS NULL OR left_memory.valid_from<=?) " - "AND (left_memory.valid_to IS NULL OR ? Optional[str]: return file_nodes.get(file_name) for edge in self.store.list_code_edges( - rid, limit=edge_cap, layers=selected_graph_layers + rid, limit=edge_cap, layers=selected_graph_layers, + flt=code_filter, ): if len(edgs) >= edge_cap: break @@ -5548,14 +6824,6 @@ def code_endpoint(value: str, file_hint: str = "") -> Optional[str]: code_links = self.store.list_code_memory_links( rid, limit=edge_cap, flt=code_filter ) - # Batched: up to `limit` (<=5000) individual get_memory() calls here - # was the dominant cost of an include_code=True request. Collect the - # candidate ids first and resolve them in one IN (...) query - # (Store.get_memories) — same liveness/limit checks below, just no - # per-row round trip. - candidate_ids = [link.get("memory_id") for link in code_links - if link.get("memory_id")] - memories_by_id = self.store.get_memories(candidate_ids) for link in code_links: if len(edgs) >= edge_cap: break @@ -5564,14 +6832,16 @@ def code_endpoint(value: str, file_hint: str = "") -> Optional[str]: if not code_id or not memory_id: continue if memory_id not in linked_memory_ids and len(entity_rows) < limit: - memory = memories_by_id.get(memory_id) - if memory and memory.expired_at is None and memory.valid_to is None: - entity_rows.append({ - "id": memory_id, - "name": memory.title or memory.content[:80] or memory_id, - "etype": f"memory_{memory.mtype.value}", - }) - linked_memory_ids.add(memory_id) + # ``list_code_memory_links`` already applied the exact + # scope/world/system filter to the joined memory. Re-reading + # it through current-only ``get_memories`` would silently + # drop a valid historical bridge. + entity_rows.append({ + "id": memory_id, + "name": link.get("title") or memory_id, + "etype": f"memory_{link.get('mtype') or 'semantic'}", + }) + linked_memory_ids.add(memory_id) if memory_id in linked_memory_ids: edgs.append({ "src": code_id, "dst": memory_id, @@ -5584,6 +6854,7 @@ def code_endpoint(value: str, file_hint: str = "") -> Optional[str]: [GraphLayer(layer) for layer in selected_layers] if selected_layers else None ), + flt=code_filter, ): if len(edgs) >= edge_cap: break @@ -5594,13 +6865,23 @@ def code_endpoint(value: str, file_hint: str = "") -> Optional[str]: "reason": link.get("reason") or "", }) payload = build_graph_payload(ws, entity_rows, edgs) - payload["unified"] = bool(include_live_code) + payload["unified"] = bool( + include_code + and (len(entity_rows) > code_node_start or len(edgs) > code_edge_start) + ) payload["repos"] = repo_names payload["meta"] = { "nodes_available": max(visible_total, len(entity_rows)), "nodes_complete": len(entity_rows) >= visible_total, "mode": "full" if full else "overview", } + if temporal_requested: + payload["meta"].update({ + "as_of": as_of, + "valid_at": valid_at, + "known_at": known_at, + "historical": True, + }) return payload def _should_backfill_graph(self, wid: str, has_entities: bool) -> bool: @@ -5738,15 +7019,83 @@ def stats(self, *, workspace: Optional[str] = None) -> dict: } -def _filter(workspace_id, repo_id, mtypes, as_of, graph_layers=None, *, session_id=None): +def _filter(workspace_id, repo_id, mtypes, as_of, graph_layers=None, *, session_id=None, + valid_at=None, known_at=None): from engraphis.core.interfaces import SearchFilter return SearchFilter( workspace_id=workspace_id, repo_id=repo_id, session_id=session_id, mtypes=mtypes, graph_layers=graph_layers, as_of=as_of, + valid_at=valid_at, known_at=known_at, include_ancestors=True, ) +def _compact_provenance(value: Any) -> dict: + """Return bounded provenance identity without copying source payload details.""" + if not isinstance(value, dict): + return {} + keys = ("source", "source_kind", "trusted", "kind", "origin") + return {key: value[key] for key in keys if key in value} + + +def _empty_recall(query: str, *, token_budget: int, response_mode: str, + retrieval_profile: str, valid_at: Optional[float], + known_at: Optional[float], note: str) -> dict: + """Stable empty response for unknown scopes, including additive v2 accounting.""" + return { + "query": query, + "count": 0, + "context": "", + "memories": [], + "packed_sources": [], + "usage": { + "budget_tokens": token_budget, + "context_tokens": 0, + "source_tokens": 0, + "saved_tokens": 0, + "savings_ratio": 0.0, + "packed_count": 0, + "omitted_count": 0, + "token_counter": "engraphis.regex.v1", + }, + "valid_at": valid_at, + "known_at": known_at, + "historical": valid_at is not None or known_at is not None, + "retrieval_profile": retrieval_profile, + "response_mode": response_mode, + "note": note, + } + + +def _empty_grounded(query: str, *, reason: str, token_budget: int, + response_mode: str, retrieval_profile: str, + valid_at: Optional[float], known_at: Optional[float]) -> dict: + payload = _empty_recall( + query, + token_budget=token_budget, + response_mode=response_mode, + retrieval_profile=retrieval_profile, + valid_at=valid_at, + known_at=known_at, + note=reason, + ) + payload.pop("count", None) + payload.pop("context", None) + payload.pop("memories", None) + payload.pop("note", None) + payload.update({ + "grounded": False, + "abstained": True, + "answer": "", + "support": 0.0, + "synthesized": False, + "citations": [], + "reason": reason, + }) + payload["usage"]["answer_tokens"] = 0 + return payload + + def _mem_to_dict(rec: Any) -> dict: """Plain, JSON-able projection of a ``MemoryRecord`` for why/timeline/proactive responses — mirrors the fields ``RecallEngine`` already exposes in recall chunks.""" @@ -5754,7 +7103,9 @@ def _mem_to_dict(rec: Any) -> dict: "id": rec.id, "title": rec.title, "content": rec.content, "summary": rec.summary, "scope": rec.scope.value, "mtype": rec.mtype.value, "repo_id": rec.repo_id, "importance": rec.importance, "pinned": rec.pinned, + "subject_key": rec.subject_key, "claim_kind": rec.claim_kind, "valid_from": rec.valid_from, "valid_to": rec.valid_to, + "valid_to_recorded_at": rec.valid_to_recorded_at, "ingested_at": rec.ingested_at, "expired_at": rec.expired_at, "provenance": rec.provenance, } diff --git a/tests/test_backends_factories.py b/tests/test_backends_factories.py index 0a425b70..5826e0ec 100644 --- a/tests/test_backends_factories.py +++ b/tests/test_backends_factories.py @@ -38,6 +38,24 @@ def test_embedder_factory_falls_back_offline(monkeypatch): assert isinstance(get_embedder("definitely-not-a-real-model-xyz", 128), DeterministicEmbedder) +def test_embedder_factory_forwards_an_immutable_model_revision(monkeypatch): + import engraphis.backends.embedder_st as embedder_st + + captured = {} + + class _PinnedEmbedder: + dim = 128 + + def __init__(self, model_name, *, revision=None): + captured.update(model_name=model_name, revision=revision) + + monkeypatch.setattr(embedder_st, "SentenceTransformerEmbedder", _PinnedEmbedder) + result = get_embedder("Qwen/example", 128, revision="a" * 40) + + assert isinstance(result, _PinnedEmbedder) + assert captured == {"model_name": "Qwen/example", "revision": "a" * 40} + + def test_deterministic_embedder_preserves_legacy_feature_hash_mapping(): """Changing the feature-hash algorithm would invalidate existing local vectors.""" vectors = DeterministicEmbedder(dim=64).embed(["alpha beta graph", "offline mapping 123"]) diff --git a/tests/test_bitemporal_recall.py b/tests/test_bitemporal_recall.py new file mode 100644 index 00000000..df607b04 --- /dev/null +++ b/tests/test_bitemporal_recall.py @@ -0,0 +1,314 @@ +import pytest + +from engraphis.core.engine import MemoryEngine +from engraphis.core.interfaces import Edge, Node, SearchFilter + + +def _engine_with_historical_memory(): + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("w") + repo_id = engine.store.get_or_create_repo(workspace_id, "r") + memory_id = engine.remember( + "The service uses a TLS certificate issued by Aurora.", + workspace_id=workspace_id, + repo_id=repo_id, + valid_from=100.0, + resolve_conflicts=False, + ) + engine.store.conn.execute( + "UPDATE memories SET ingested_at=? WHERE id=?", (200.0, memory_id) + ) + engine.store.conn.commit() + return engine, workspace_id, repo_id, memory_id + + +def test_search_filter_preserves_legacy_positional_include_ancestors(): + flt = SearchFilter("w", "r", None, None, None, None, 123.0, True) + assert flt.as_of == flt.valid_at == 123.0 + assert flt.include_ancestors is True + assert flt.known_at is None + + +def test_ordinary_recall_uses_one_effective_snapshot_without_becoming_historical( + monkeypatch, +): + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("w") + memory_id = engine.remember( + "The snapshot marker is blue.", + workspace_id=workspace_id, + resolve_conflicts=False, + ) + engine.store.conn.execute( + "UPDATE memories SET valid_from=100, ingested_at=100 WHERE id=?", + (memory_id,), + ) + engine.store.conn.commit() + monkeypatch.setattr("engraphis.core.recall.now_ts", lambda: 123.0) + + result = engine.recall( + "snapshot marker", + workspace_id=workspace_id, + reinforce=True, + ) + + assert result.valid_at == result.known_at == 123.0 + assert result.historical is False + assert engine.store.get_memory(memory_id).access_count == 1 + + +def test_valid_at_and_known_at_keep_future_knowledge_out_of_recall(): + engine, workspace_id, repo_id, memory_id = _engine_with_historical_memory() + + before_known = engine.recall( + "Which certificate issuer does the service use?", + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=150.0, + known_at=199.0, + ) + once_known = engine.recall( + "Which certificate issuer does the service use?", + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=150.0, + known_at=200.0, + ) + + assert before_known.chunks == [] + assert [chunk["id"] for chunk in once_known.chunks] == [memory_id] + + +def test_historical_recall_is_observational(): + engine, workspace_id, repo_id, memory_id = _engine_with_historical_memory() + before = engine.store.get_memory(memory_id) + + result = engine.recall( + "Which certificate issuer does the service use?", + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=150.0, + known_at=200.0, + ) + after = engine.store.get_memory(memory_id) + + assert [chunk["id"] for chunk in result.chunks] == [memory_id] + assert after.access_count == before.access_count + assert after.last_access == before.last_access + assert after.stability == before.stability + + +def test_system_expiry_is_evaluated_at_known_at_not_the_present(): + engine, workspace_id, repo_id, memory_id = _engine_with_historical_memory() + engine.store.conn.execute( + "UPDATE memories SET expired_at=? WHERE id=?", (300.0, memory_id) + ) + engine.store.conn.commit() + + before_expiry = engine.store.list_memories(SearchFilter( + workspace_id=workspace_id, valid_at=150.0, known_at=299.0, + )) + after_expiry = engine.store.list_memories(SearchFilter( + workspace_id=workspace_id, valid_at=150.0, known_at=300.0, + )) + + assert [record.id for record in before_expiry] == [memory_id] + assert after_expiry == [] + + +def test_retroactive_supersession_is_visible_only_after_it_was_learned(): + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("w") + repo_id = engine.store.get_or_create_repo(workspace_id, "r") + old_id = engine.remember( + "The production endpoint was alpha.", + workspace_id=workspace_id, + repo_id=repo_id, + valid_from=100.0, + resolve_conflicts=False, + ) + engine.store.conn.execute( + "UPDATE memories SET ingested_at=100 WHERE id=?", (old_id,) + ) + engine.store.conn.commit() + engine.store.close_validity(old_id, at=200.0) + engine.store.conn.execute( + "UPDATE memories SET valid_to_recorded_at=300 WHERE id=?", (old_id,) + ) + new_id = engine.remember( + "The production endpoint was beta.", + workspace_id=workspace_id, + repo_id=repo_id, + valid_from=200.0, + resolve_conflicts=False, + ) + engine.store.conn.execute( + "UPDATE memories SET ingested_at=300 WHERE id=?", (new_id,) + ) + engine.store.conn.commit() + + believed_before_correction = engine.recall( + "What was the production endpoint?", + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=250.0, + known_at=250.0, + ) + corrected_view = engine.recall( + "What was the production endpoint?", + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=250.0, + known_at=350.0, + ) + past_world_after_correction = engine.recall( + "What was the production endpoint?", + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=150.0, + known_at=350.0, + ) + + assert [chunk["id"] for chunk in believed_before_correction.chunks] == [old_id] + assert [chunk["id"] for chunk in corrected_view.chunks] == [new_id] + assert [chunk["id"] for chunk in past_world_after_correction.chunks] == [old_id] + + +def test_graph_edges_neighbors_and_supports_share_bitemporal_visibility(): + engine, workspace_id, repo_id, memory_id = _engine_with_historical_memory() + store = engine.store + alpha = store.upsert_entity(Node( + id="", name="alpha", workspace_id=workspace_id, repo_id=repo_id, + )) + beta = store.upsert_entity(Node( + id="", name="beta", workspace_id=workspace_id, repo_id=repo_id, + )) + edge_id = store.upsert_edge(Edge( + id="", src=alpha, dst=beta, relation="depends_on", + workspace_id=workspace_id, repo_id=repo_id, + valid_from=100.0, ingested_at=200.0, expired_at=300.0, + provenance={"memory_id": memory_id}, + )) + before_known = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=150.0, known_at=199.0, + ) + visible = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=150.0, known_at=200.0, + ) + expired = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=150.0, known_at=300.0, + ) + + assert store.edges_in_scope(before_known) == [] + assert [edge.id for edge in store.edges_in_scope(visible)] == [edge_id] + assert store.edges_in_scope(expired) == [] + assert store.neighbors([alpha], flt=before_known) == [] + assert [edge.id for edge in store.neighbors([alpha], flt=visible)] == [edge_id] + assert store.neighbors([alpha], flt=expired) == [] + assert store.edge_supports_in_scope([edge_id], flt=before_known) == [] + assert len(store.edge_supports_in_scope([edge_id], flt=visible)) == 1 + assert store.edge_supports_in_scope([edge_id], flt=expired) == [] + + +def test_retroactive_edge_closure_does_not_leak_before_it_was_known(): + engine, workspace_id, repo_id, memory_id = _engine_with_historical_memory() + store = engine.store + edge_id = store.upsert_edge(Edge( + id="edge_history", + src="alpha", + dst="beta", + relation="depends_on", + workspace_id=workspace_id, + repo_id=repo_id, + valid_from=100.0, + ingested_at=100.0, + provenance={"memory_id": memory_id}, + )) + store.invalidate_edge(edge_id, at=200.0) + store.conn.execute( + "UPDATE edges SET valid_to_recorded_at=300 WHERE id=?", (edge_id,) + ) + store.conn.execute( + "UPDATE edge_supports SET valid_to_recorded_at=300 WHERE edge_id=?", (edge_id,) + ) + store.conn.commit() + + before_correction = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=250.0, known_at=250.0, + ) + after_correction = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=250.0, known_at=350.0, + ) + earlier_world = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=150.0, known_at=350.0, + ) + + assert [edge.id for edge in store.edges_in_scope(before_correction)] == [edge_id] + assert store.edges_in_scope(after_correction) == [] + assert [edge.id for edge in store.edges_in_scope(earlier_world)] == [edge_id] + assert store.edge_supports_in_scope([edge_id], flt=before_correction) + assert store.edge_supports_in_scope([edge_id], flt=after_correction) == [] + + +def test_memory_links_share_bitemporal_visibility_and_do_not_leak_future_associations(): + """A late direct-link must not change a historical graph walk.""" + engine, workspace_id, repo_id, memory_id = _engine_with_historical_memory() + other_id = engine.remember( + "Aurora certificate operations have a migration runbook.", + workspace_id=workspace_id, + repo_id=repo_id, + valid_from=100.0, + resolve_conflicts=False, + ) + engine.store.conn.execute( + "UPDATE memories SET ingested_at=100 WHERE id=?", (other_id,) + ) + engine.store.add_link( + memory_id, other_id, "related", valid_from=100.0, ingested_at=200.0, + ) + engine.store.conn.commit() + + before_known = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=150.0, known_at=199.0, + ) + visible = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=150.0, known_at=200.0, + ) + assert engine.store.links_among([memory_id, other_id], flt=before_known) == [] + assert [link["relation"] for link in engine.store.links_among( + [memory_id, other_id], flt=visible, + )] == ["related"] + + engine.store.conn.execute( + "UPDATE mem_links SET valid_to=200, valid_to_recorded_at=300 " + "WHERE a=? AND b=?", (memory_id, other_id), + ) + engine.store.conn.commit() + believed_before_closure = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=250.0, known_at=250.0, + ) + corrected_view = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=250.0, known_at=350.0, + ) + earlier_world = SearchFilter( + workspace_id=workspace_id, repo_id=repo_id, valid_at=150.0, known_at=350.0, + ) + assert engine.store.links_among([memory_id, other_id], flt=believed_before_closure) + assert engine.store.links_among([memory_id, other_id], flt=corrected_view) == [] + assert engine.store.links_among([memory_id, other_id], flt=earlier_world) + + +def test_as_of_is_the_valid_at_compatibility_alias_and_conflicts_are_rejected(): + compatible = SearchFilter(as_of=150.0) + assert compatible.valid_at == compatible.as_of == 150.0 + assert compatible.historical + + with pytest.raises(ValueError, match="as_of and valid_at must match"): + SearchFilter(as_of=100.0, valid_at=101.0) + + +@pytest.mark.parametrize("field", ["as_of", "valid_at", "known_at"]) +def test_temporal_filter_anchors_must_be_finite(field): + for invalid in (float("nan"), True): + with pytest.raises(ValueError, match=field + " must be a finite timestamp"): + SearchFilter(**{field: invalid}) diff --git a/tests/test_canonical_export.py b/tests/test_canonical_export.py new file mode 100644 index 00000000..41edf7d1 --- /dev/null +++ b/tests/test_canonical_export.py @@ -0,0 +1,456 @@ +"""Deterministic workspace-export evidence.""" +from __future__ import annotations + +import hashlib +import json + +from engraphis.service import MemoryService, set_current_user + + +def _digest_payload(export: dict) -> str: + payload = dict(export) + expected = payload.pop("sha256") + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ).encode("utf-8") + assert hashlib.sha256(encoded).hexdigest() == expected + return expected + + +def test_unchanged_workspace_has_identical_canonical_export_and_digest(): + service = MemoryService.create(":memory:") + service.remember( + "Deployments require signed release tags.", + workspace="acme", + repo="api", + ) + + first = service.export_workspace(workspace="acme", canonical=True) + second = service.export_workspace(workspace="acme", canonical=True) + + assert first == second + assert first["canonical"] is True + assert "exported_at" not in first + assert _digest_payload(first) == _digest_payload(second) + + +def test_default_workspace_export_is_v2_and_keeps_timestamped_compatibility_fields(): + service = MemoryService.create(":memory:") + service.remember("A durable fact.", workspace="acme") + + exported = service.export_workspace(workspace="acme") + + assert exported["format"] == "engraphis-export/2" + assert "exported_at" in exported + assert "sha256" not in exported + assert exported["counts"]["memories"] == 1 + assert exported["completeness"]["durable_workspace_state"] is True + assert exported["completeness"]["receipts"] is True + assert exported["receipt_verification"]["valid"] is True + + +def test_canonical_digest_covers_graph_and_code_state(): + service = MemoryService.create(":memory:") + memory = service.remember( + "The API delegates parsing to load_config.", + workspace="acme", + repo="api", + ) + wid = service.store.get_or_create_workspace("acme") + rid = service.store.get_or_create_repo(wid, "api") + baseline = service.export_workspace(workspace="acme", canonical=True)["sha256"] + + service.store.conn.executemany( + "INSERT INTO entities(id, workspace_id, repo_id, name, etype, created_at) " + "VALUES (?,?,?,?,?,?)", + [ + ("ent_export_api", wid, rid, "API", "module", 1.0), + ("ent_export_config", wid, rid, "Config", "module", 1.0), + ], + ) + service.store.conn.execute( + "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " + "valid_from, ingested_at, provenance) VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + "edg_export_graph", wid, rid, "ent_export_api", "ent_export_config", + "depends_on", "causal", 1.0, 1.0, + json.dumps({"memory_id": memory["id"]}), + ), + ) + service.store.conn.execute( + "INSERT INTO edge_supports(edge_id, memory_id, source_kind, confidence, " + "valid_from, ingested_at, provenance) VALUES (?,?,?,?,?,?,?)", + ( + "edg_export_graph", memory["id"], "manual", 1.0, 1.0, 1.0, + json.dumps({"memory_id": memory["id"]}), + ), + ) + service.store.conn.commit() + after_graph = service.export_workspace( + workspace="acme", canonical=True + )["sha256"] + assert after_graph != baseline + + service.store.conn.execute( + "INSERT INTO symbols(id, repo_id, kind, name, fqname, file, lang, " + "valid_from, ingested_at) VALUES (?,?,?,?,?,?,?,?,?)", + ( + "sym_export_load_config", rid, "function", "load_config", + "config.load_config", "config.py", "python", 1.0, 1.0, + ), + ) + service.store.conn.execute( + "INSERT INTO code_files(repo_id, file, lang, content_hash, size_bytes, " + "mtime_ns, backend, indexed_at) VALUES (?,?,?,?,?,?,?,?)", + (rid, "config.py", "python", "sha256:test", 42, 1, "test", 1.0), + ) + service.store.conn.execute( + "INSERT INTO code_memory_links(id, repo_id, symbol_id, memory_id, relation, " + "confidence, created_at, valid_from, ingested_at) " + "VALUES (?,?,?,?,?,?,?,?,?)", + ( + "edg_export_code_memory", rid, "sym_export_load_config", memory["id"], + "mentions", 1.0, 1.0, 1.0, 1.0, + ), + ) + service.store.conn.commit() + after_code = service.export_workspace( + workspace="acme", canonical=True + )["sha256"] + + assert after_code != after_graph + exported = service.export_workspace(workspace="acme", canonical=True) + assert exported["counts"]["entities"] == 2 + assert exported["counts"]["edges"] == 1 + assert exported["counts"]["edge_supports"] == 1 + assert exported["counts"]["symbols"] == 1 + assert exported["counts"]["code_files"] == 1 + assert exported["counts"]["code_memory_links"] == 1 + + +def test_member_export_filters_private_session_derivatives_but_admin_export_is_complete(): + service = MemoryService.create(":memory:") + try: + set_current_user({ + "id": "usr_alice", + "email": "alice@example.test", + "role": "member", + }) + service.create_workspace("shared", visibility="shared", confirmed=True) + shared = service.remember( + "Shared release guidance.", workspace="shared", repo="api", + scope="repo", + ) + alice_session = service.start_session( + "shared", repo="api", agent="codex", goal="internal material" + ) + private = service.remember( + "ALICE_EXPORT_PRIVATE_MARKER", + workspace="shared", + repo="api", + session_id=alice_session["session_id"], + scope="session", + ) + service.record_event( + "private_note", + "ALICE_EXPORT_EVENT_MARKER", + workspace="shared", + repo="api", + session_id=alice_session["session_id"], + refs=[private["id"]], + ) + wid = service.store.get_or_create_workspace("shared") + rid = service.store.get_or_create_repo(wid, "api") + service.store.conn.execute( + "UPDATE memories SET session_id=?, metadata=?, provenance=? WHERE id=?", + ( + alice_session["session_id"], + json.dumps({"related": private["id"]}), + json.dumps({"memory_id": private["id"]}), + shared["id"], + ), + ) + service.store.conn.executemany( + "INSERT INTO entities(id, workspace_id, repo_id, name, etype, created_at) " + "VALUES (?,?,?,?,?,?)", + [ + ("ent_alice_secret", wid, rid, "ALICE_EXPORT_ENTITY_MARKER", "secret", 1.0), + ("ent_alice_target", wid, rid, "Private target", "secret", 1.0), + ], + ) + service.store.conn.execute( + "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " + "valid_from, ingested_at, provenance) VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + "edg_alice_private", wid, rid, "ent_alice_secret", "ent_alice_target", + "related", "semantic", 1.0, 1.0, + json.dumps({"memory_id": private["id"]}), + ), + ) + service.store.conn.execute( + "INSERT INTO edge_supports(edge_id, memory_id, source_kind, confidence, " + "valid_from, ingested_at, provenance) VALUES (?,?,?,?,?,?,?)", + ( + "edg_alice_private", private["id"], "manual", 1.0, 1.0, 1.0, + json.dumps({"memory_id": private["id"]}), + ), + ) + service.store.conn.execute( + "INSERT INTO memory_entities(id, memory_id, entity_id, workspace_id, repo_id, " + "source_kind, confidence, valid_from, ingested_at, provenance) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + "edg_alice_incidence", private["id"], "ent_alice_secret", wid, rid, + "manual", 1.0, 1.0, 1.0, + json.dumps({"memory_id": private["id"]}), + ), + ) + service.store.conn.execute( + "INSERT INTO mem_links(a, b, relation, layer, reason, created_at) " + "VALUES (?,?,?,?,?,?)", + ( + private["id"], shared["id"], "related", "semantic", + "ALICE_EXPORT_LINK_MARKER", 1.0, + ), + ) + service.store.conn.execute( + "INSERT INTO symbols(id, repo_id, kind, name, fqname, file, lang, " + "valid_from, ingested_at) VALUES (?,?,?,?,?,?,?,?,?)", + ( + "sym_shared_export", rid, "function", "shared_export", + "api.shared_export", "api.py", "python", 1.0, 1.0, + ), + ) + service.store.conn.execute( + "INSERT INTO code_memory_links(id, repo_id, symbol_id, memory_id, relation, " + "confidence, created_at, valid_from, ingested_at) " + "VALUES (?,?,?,?,?,?,?,?,?)", + ( + "edg_alice_code_link", rid, "sym_shared_export", private["id"], + "mentions", 1.0, 1.0, 1.0, 1.0, + ), + ) + service.store.conn.commit() + + set_current_user({ + "id": "usr_bob", + "email": "bob@example.test", + "role": "member", + }) + bob_export = service.export_workspace(workspace="shared", canonical=True) + serialized = json.dumps(bob_export, sort_keys=True) + assert bob_export["visibility"] == "principal" + assert private["id"] not in serialized + assert alice_session["session_id"] not in serialized + assert "alice@example.test" not in serialized + assert "ALICE_EXPORT_PRIVATE_MARKER" not in serialized + assert "ALICE_EXPORT_EVENT_MARKER" not in serialized + assert "ALICE_EXPORT_ENTITY_MARKER" not in serialized + assert "ALICE_EXPORT_LINK_MARKER" not in serialized + assert all( + row["memory_id"] != private["id"] + for row in bob_export["edge_supports"] + ) + assert all( + row["memory_id"] != private["id"] + for row in bob_export["memory_entities"] + ) + assert all( + private["id"] not in {row["a"], row["b"]} + for row in bob_export["memory_links"] + ) + assert all( + row["memory_id"] != private["id"] + for row in bob_export["code_memory_links"] + ) + exported_shared = next( + row for row in bob_export["memories"] if row["id"] == shared["id"] + ) + assert exported_shared["session_id"] is None + assert json.loads(exported_shared["metadata"]) == {} + assert json.loads(exported_shared["provenance"]) == {} + + set_current_user({ + "id": "usr_admin", + "email": "admin@example.test", + "role": "admin", + }) + admin_export = service.export_workspace(workspace="shared", canonical=True) + assert admin_export["visibility"] == "workspace" + assert "ALICE_EXPORT_PRIVATE_MARKER" in json.dumps(admin_export) + assert any( + row["memory_id"] == private["id"] + for row in admin_export["edge_supports"] + ) + assert any( + row["memory_id"] == private["id"] + for row in admin_export["memory_entities"] + ) + assert any( + private["id"] in {row["a"], row["b"]} + for row in admin_export["memory_links"] + ) + assert any( + row["memory_id"] == private["id"] + for row in admin_export["code_memory_links"] + ) + finally: + set_current_user(None) + + +def test_canonical_export_includes_more_than_ten_thousand_receipts(): + service = MemoryService.create(":memory:") + wid = service.store.get_or_create_workspace("acme") + previous = "" + rows = [] + for index in range(10_001): + receipt_id = f"rcpt_{index:026d}" + payload = json.dumps( + { + "version": 1, + "id": receipt_id, + "ts_ms": index, + "operation": "recall", + "scope_digest": "0" * 24, + "actor_digest": "1" * 16, + "target_count": 0, + "status": "ok", + "metadata": {}, + "prev_hash": previous, + }, + sort_keys=True, + separators=(",", ":"), + ) + receipt_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + rows.append( + ( + receipt_id, index / 1000.0, "recall", wid, "", index + 1, + "scope", "actor", 0, "ok", payload, previous, receipt_hash, + ) + ) + previous = receipt_hash + service.store.conn.executemany( + "INSERT INTO operation_receipts(id, ts, operation, workspace_id, repo_id, " + "sequence, scope_digest, actor, target_count, status, payload, prev_hash, " + "receipt_hash) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + rows, + ) + service.store.conn.execute( + "INSERT INTO receipt_chain_heads(workspace_id, receipt_count, head_hash, " + "integrity_error, updated_at) VALUES (?,?,?,?,?)", + (wid, len(rows), previous, "", 10.001), + ) + service.store.conn.commit() + + exported = service.export_workspace(workspace="acme", canonical=True) + receipt_export = service.export_receipts(workspace="acme") + assert exported["counts"]["receipts"] == 10_001 + assert len(exported["receipts"]) == 10_001 + assert len(receipt_export["entries"]) == 10_001 + assert exported["receipt_verification"]["valid"] is True + assert receipt_export["verification"]["valid"] is True + + original_digest = exported["sha256"] + service.store.conn.execute( + "UPDATE operation_receipts SET receipt_hash=? WHERE id=?", + ("0" * 64, "rcpt_" + ("0" * 26)), + ) + service.store.conn.commit() + tampered = service.export_workspace(workspace="acme", canonical=True) + assert tampered["sha256"] != original_digest + assert tampered["receipt_verification"]["valid"] is False + + +def test_receipt_export_redacts_malformed_payload_and_anchor_text(): + service = MemoryService.create(":memory:") + stored = service.remember("A safe fact.", workspace="acme") + wid = service.store.get_or_create_workspace("acme") + markers = { + "id": "POISONED_RECEIPT_ID_PRIVATE_MARKER", + "hash": "POISONED_RECEIPT_HASH_PRIVATE_MARKER", + "payload": "POISONED_RECEIPT_PAYLOAD_PRIVATE_MARKER", + "anchor": "POISONED_RECEIPT_ANCHOR_PRIVATE_MARKER", + "integrity": "POISONED_RECEIPT_INTEGRITY_PRIVATE_MARKER", + "prev": "POISONED_RECEIPT_PREV_PRIVATE_MARKER", + "count": "POISONED_RECEIPT_COUNT_PRIVATE_MARKER", + } + raw = json.dumps({"secret": markers["payload"]}) + service.store.conn.execute( + "UPDATE operation_receipts SET id=?, payload=?, receipt_hash=? WHERE id=?", + (markers["id"], raw, markers["hash"], stored["receipt"]["id"]), + ) + service.store.conn.execute( + "UPDATE receipt_chain_heads SET receipt_count=?, head_hash=?, integrity_error=? " + "WHERE workspace_id=?", + (markers["count"], markers["anchor"], markers["integrity"], wid), + ) + service.store.conn.commit() + + exported = service.export_workspace(workspace="acme", canonical=True) + receipt_export = service.export_receipts(workspace="acme") + receipt_log = service.receipt_log(workspace="acme") + direct_verification = service.verify_receipts(workspace="acme") + + other = service.remember("Another safe fact.", workspace="other") + service.store.conn.execute( + "UPDATE operation_receipts SET prev_hash=? WHERE id=?", + (markers["prev"], other["receipt"]["id"]), + ) + service.store.conn.commit() + other_export = service.export_receipts(workspace="other") + + encoded = json.dumps({ + "workspace": exported, + "receipts": receipt_export, + "receipt_log": receipt_log, + "verification": direct_verification, + "other": other_export, + }) + assert all(marker not in encoded for marker in markers.values()) + assert exported["receipts"][0]["invalid_payload"] is True + assert "raw_payload" not in exported["receipts"][0] + assert exported["receipts"][0]["id"].startswith("redacted_sha256:") + assert exported["receipts"][0]["hash"].startswith("redacted_sha256:") + assert receipt_export["verification"]["head"].startswith("redacted_sha256:") + assert all( + not error["id"] or error["id"].startswith("redacted_sha256:") + for error in receipt_export["verification"]["errors"] + ) + assert other_export["entries"][0]["prev_hash"].startswith("redacted_sha256:") + assert exported["receipt_chain"]["integrity_error"].startswith("redacted_sha256:") + assert exported["receipt_chain"]["head_hash"].startswith("redacted_sha256:") + assert exported["receipt_chain"]["receipt_count"] is None + assert direct_verification["valid"] is False + + +def test_receipt_export_accepts_declared_terminal_statuses(): + service = MemoryService.create(":memory:") + wid = service.store.get_or_create_workspace("acme") + service.store.record_receipt("sync", workspace_id=wid, status="failed") + service.store.record_receipt("sync", workspace_id=wid, status="cancelled") + + exported = service.export_receipts(workspace="acme") + assert [row["status"] for row in exported["entries"]] == [ + "failed", "cancelled", + ] + assert all("invalid_payload" not in row for row in exported["entries"]) + + +def test_workspace_export_owns_only_the_transaction_it_starts(): + service = MemoryService.create(":memory:") + service.remember("A safe fact.", workspace="acme") + + service.export_workspace(workspace="acme", canonical=True) + assert service.store.conn.in_transaction is False + + service.store.conn.execute("BEGIN") + try: + service.export_workspace(workspace="acme", canonical=True) + service.export_receipts(workspace="acme") + assert service.store.conn.in_transaction is True + finally: + service.store.conn.rollback() diff --git a/tests/test_code_recall_arm.py b/tests/test_code_recall_arm.py new file mode 100644 index 00000000..2d534fe9 --- /dev/null +++ b/tests/test_code_recall_arm.py @@ -0,0 +1,160 @@ +"""First-class code-symbol recall arm.""" +from __future__ import annotations + +import pytest + +from engraphis.core.engine import MemoryEngine +from engraphis.core.interfaces import SearchFilter + + +def test_code_profile_bridges_symbols_to_scoped_memories(tmp_path): + (tmp_path / "deploy.py").write_text( + "def deploy_release():\n return 'ok'\n", + encoding="utf-8", + ) + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("acme") + repo_id = engine.store.get_or_create_repo(workspace_id, "api") + engine.index_repo(repo_id, str(tmp_path), prefer="regex") + memory_id = engine.remember( + "deploy_release requires a signed tag and a successful backup.", + workspace_id=workspace_id, + repo_id=repo_id, + ) + + result = engine.recall_engine.recall( + "What calls deploy_release()?", + SearchFilter( + workspace_id=workspace_id, + repo_id=repo_id, + include_ancestors=True, + ), + k=5, + reinforce=False, + retrieval_profile="code", + diagnostics=True, + ) + + assert memory_id in {chunk["id"] for chunk in result.chunks} + detail = next(item for item in result.retrieval_trace if item["id"] == memory_id) + assert "code" in detail["arms"] + assert detail["raw"]["code"] > 0 + + +def test_auto_profile_selects_code_without_changing_balanced_default(tmp_path): + (tmp_path / "worker.py").write_text( + "def process_queue():\n return None\n", + encoding="utf-8", + ) + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("acme") + repo_id = engine.store.get_or_create_repo(workspace_id, "worker") + engine.index_repo(repo_id, str(tmp_path), prefer="regex") + engine.remember( + "process_queue drains the durable retry queue.", + workspace_id=workspace_id, + repo_id=repo_id, + ) + flt = SearchFilter( + workspace_id=workspace_id, + repo_id=repo_id, + include_ancestors=True, + ) + + balanced = engine.recall_engine.recall( + "process_queue()", flt, reinforce=False + ) + automatic = engine.recall_engine.recall( + "process_queue()", flt, reinforce=False, retrieval_profile="auto" + ) + + assert balanced.retrieval_profile == "balanced" + assert automatic.retrieval_profile == "code" + + +def test_historical_code_arm_fails_closed_for_legacy_store_methods(monkeypatch): + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("acme") + repo_id = engine.store.get_or_create_repo(workspace_id, "api") + calls = [] + + def legacy_search_symbols(repo, query, *, limit=20): + calls.append((repo, query, limit)) + return [{"id": "sym_current", "name": "deploy", "fqname": "deploy"}] + + monkeypatch.setattr(engine.store, "search_symbols", legacy_search_symbols) + scores = engine.recall_engine._code_arm( + "deploy()", + SearchFilter( + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=10.0, + known_at=10.0, + ), + 10, + ) + + assert scores == {} + assert calls == [], "historical reads must never retry without the temporal filter" + + +def test_code_arm_does_not_mask_type_errors_from_temporal_store(monkeypatch): + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("acme") + repo_id = engine.store.get_or_create_repo(workspace_id, "api") + + def broken_search_symbols(repo, query, *, limit=20, flt=None): + raise TypeError("implementation bug") + + monkeypatch.setattr(engine.store, "search_symbols", broken_search_symbols) + with pytest.raises(TypeError, match="implementation bug"): + engine.recall_engine._code_arm( + "deploy()", + SearchFilter(workspace_id=workspace_id, repo_id=repo_id), + 10, + ) + + +def test_code_arm_batches_memory_lookup_for_many_symbols(monkeypatch): + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("acme") + repo_id = engine.store.get_or_create_repo(workspace_id, "api") + symbols = [ + {"id": f"sym_{index:03d}", "name": f"DeployTarget{index}", + "fqname": f"api.DeployTarget{index}"} + for index in range(100) + ] + batch_calls = [] + + monkeypatch.setattr( + engine.store, + "search_symbols", + lambda repo, query, *, limit=20, flt=None: symbols, + ) + monkeypatch.setattr( + engine.store, + "list_code_edges", + lambda repo, *, limit=None, layers=None, flt=None: [], + ) + monkeypatch.setattr( + engine.store, + "memories_for_symbol", + lambda *args, **kwargs: pytest.fail("per-symbol lookup must not run"), + ) + + def batched(repo, symbol_ids, *, flt=None, limit=20): + batch_calls.append((repo, list(symbol_ids), limit)) + return { + symbol_id: [{"id": f"mem_{symbol_id}", "confidence": 1.0}] + for symbol_id in symbol_ids + } + + monkeypatch.setattr(engine.store, "memories_for_symbols", batched) + scores = engine.recall_engine._code_arm( + "DeployTarget()", SearchFilter(workspace_id=workspace_id, repo_id=repo_id), 50 + ) + + assert len(batch_calls) == 1 + assert batch_calls[0][0] == repo_id + assert len(batch_calls[0][1]) == 100 + assert len(scores) == 50 diff --git a/tests/test_compact_recall.py b/tests/test_compact_recall.py new file mode 100644 index 00000000..afa531a2 --- /dev/null +++ b/tests/test_compact_recall.py @@ -0,0 +1,212 @@ +"""Public compact-recall and token-accounting contracts.""" +from __future__ import annotations + +import json + +import pytest + +from engraphis.core.context import RegexTokenCounter +from engraphis.service import MemoryService, ValidationError + + +def _seed_service() -> MemoryService: + service = MemoryService.create(":memory:") + for index in range(4): + service.remember( + ( + f"Release policy evidence {index}: deployments require a signed tag and " + "a successful backup verification before production promotion. " + + "Operational rationale and audit detail remain attached to this record. " * 20 + ), + workspace="acme", + repo="api", + title=f"Release policy {index}", + resolve_conflicts=False, + ) + return service + + +def test_compact_recall_has_a_hard_budget_and_omits_duplicate_bodies(): + service = _seed_service() + + result = service.recall( + "What evidence governs release deployment?", + workspace="acme", + repo="api", + k=4, + token_budget=80, + response_mode="compact", + reinforce=False, + record_receipt=False, + ) + + assert result["usage"]["context_tokens"] == RegexTokenCounter()(result["context"]) + assert result["usage"]["context_tokens"] <= 80 + assert result["usage"]["token_counter"] == "engraphis.regex.v1" + assert result["packed_sources"] + assert all("content" not in source for source in result["memories"]) + + +def test_compact_serialized_payload_saves_at_least_half_vs_legacy_full(): + service = _seed_service() + kwargs = { + "workspace": "acme", + "repo": "api", + "k": 4, + "token_budget": 80, + "reinforce": False, + "record_receipt": False, + } + + full = service.recall( + "What evidence governs release deployment?", + response_mode="full", + **kwargs, + ) + compact = service.recall( + "What evidence governs release deployment?", + response_mode="compact", + **kwargs, + ) + count = RegexTokenCounter() + full_tokens = count(json.dumps(full, sort_keys=True)) + compact_tokens = count(json.dumps(compact, sort_keys=True)) + + assert compact_tokens <= full_tokens * 0.5 + assert compact["usage"]["savings_ratio"] > 0.5 + + +def test_receipt_records_only_privacy_safe_token_aggregates(): + service = _seed_service() + secret_query = "private-query-marker release deployment" + + service.recall( + secret_query, + workspace="acme", + repo="api", + token_budget=64, + response_mode="compact", + ) + row = service.store.conn.execute( + "SELECT payload FROM operation_receipts ORDER BY rowid DESC LIMIT 1" + ).fetchone() + payload = json.loads(row["payload"]) + metadata = payload["metadata"] + + assert secret_query not in row["payload"] + assert metadata["token_usage"]["budget_tokens"] == 64 + assert metadata["token_usage"]["context_tokens"] <= 64 + assert metadata["token_usage"]["token_counter"] == "engraphis.regex.v1" + + +def test_recall_temporal_alias_conflict_and_modes_fail_closed(): + service = _seed_service() + + with pytest.raises(ValidationError, match="must match"): + service.recall( + "release", + workspace="acme", + as_of=100.0, + valid_at=101.0, + ) + with pytest.raises(ValidationError, match="response_mode"): + service.recall("release", workspace="acme", response_mode="tiny") + with pytest.raises(ValidationError, match="retrieval_profile"): + service.recall("release", workspace="acme", retrieval_profile="magic") + + +def test_diagnostics_preserve_each_score_stage_without_changing_default_payload(): + service = _seed_service() + + normal = service.recall( + "release deployment evidence", + workspace="acme", + repo="api", + reinforce=False, + record_receipt=False, + ) + diagnostic = service.recall( + "release deployment evidence", + workspace="acme", + repo="api", + reinforce=False, + record_receipt=False, + diagnostics=True, + retrieval_profile="lexical", + ) + + assert "retrieval_trace" not in normal + assert diagnostic["retrieval_profile"] == "lexical" + assert diagnostic["retrieval_trace"] + stages = diagnostic["retrieval_trace"][0] + assert { + "raw", + "normalized", + "fusion_score", + "rerank_score", + "calibrated_score", + "arm_agreement", + } <= stages.keys() + + +def test_service_exposes_claim_identity_for_safe_supersession(): + service = MemoryService.create(":memory:") + first = service.remember( + "The configured API request ceiling is one hundred.", + workspace="acme", + subject_key="api.rate_limit", + claim_kind="configured_value", + ) + second = service.remember( + "The configured API request ceiling is five hundred.", + workspace="acme", + subject_key="api.rate_limit", + claim_kind="configured_value", + ) + + assert second["op"] == "invalidate" + assert second["superseded"] == [first["id"]] + + +def test_compact_grounded_response_does_not_repeat_cited_bodies(): + service = MemoryService.create(":memory:") + long_body = ( + "The API authenticates with PASETO v4 public tokens. " + + "This intentionally long evidence body carries bounded operational detail. " * 24 + ) + service.remember( + long_body, + workspace="acme", + repo="api", + ) + + full = service.grounded_recall( + "How does the API authenticate?", + workspace="acme", + repo="api", + min_support=0.0, + response_mode="full", + token_budget=48, + ) + compact = service.grounded_recall( + "How does the API authenticate?", + workspace="acme", + repo="api", + min_support=0.0, + response_mode="compact", + token_budget=48, + ) + + assert full["citations"] and "content" in full["citations"][0] + assert compact["citations"] and "content" not in compact["citations"][0] + assert "excerpt" not in compact["citations"][0] + assert compact["grounded"] == full["grounded"] + assert compact["answer"] == full["answer"] + assert [citation["id"] for citation in compact["citations"]] == [ + citation["id"] for citation in full["citations"] + ] + assert "PASETO" in compact["answer"] + assert long_body not in json.dumps(compact, sort_keys=True) + assert compact["usage"]["answer_tokens"] == RegexTokenCounter()(compact["answer"]) + assert compact["usage"]["answer_tokens"] <= 48 + assert compact["usage"]["context_tokens"] <= 48 diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py index 6adc190e..7a8c45cd 100644 --- a/tests/test_consolidate.py +++ b/tests/test_consolidate.py @@ -3,9 +3,9 @@ import pytest -from engraphis.core.consolidate import consolidate +from engraphis.core.consolidate import _cluster_by_subject, consolidate from engraphis.core.engine import MemoryEngine -from engraphis.core.interfaces import MemoryType, SearchFilter +from engraphis.core.interfaces import MemoryRecord, MemoryType, SearchFilter from engraphis.service import MemoryService, ValidationError @@ -25,6 +25,31 @@ def _engine_with_repeats(): return eng, wid, rid +def test_entity_clustering_uses_connected_components_not_first_link_assignment(): + memories = [ + MemoryRecord(id="mem_a", content="a"), + MemoryRecord(id="mem_b", content="b"), + MemoryRecord(id="mem_c", content="c"), + ] + + class IncidenceStore: + def list_memory_entities(self, _flt): + # A bridges X and Y. A first-link implementation splits C away. + return [ + {"memory_id": "mem_a", "entity_id": "ent_x"}, + {"memory_id": "mem_a", "entity_id": "ent_y"}, + {"memory_id": "mem_b", "entity_id": "ent_x"}, + {"memory_id": "mem_c", "entity_id": "ent_y"}, + ] + + groups = _cluster_by_subject( + memories, threshold=1.0, store=IncidenceStore(), flt=SearchFilter() + ) + assert [[memory.id for memory in group] for group in groups] == [ + ["mem_a", "mem_b", "mem_c"] + ] + + def test_service_rejects_non_finite_archive_threshold(): service = MemoryService.create(":memory:") service.create_workspace("w") @@ -501,9 +526,7 @@ def test_archive_pass_sees_transients_behind_newer_semantic_rows(monkeypatch): assert [row["id"] for row in report["archived"]] == [stale] -def test_archive_logs_index_cleanup_failure_without_leaking_exception_text( - monkeypatch, caplog -): +def test_archive_preserves_vector_for_historical_recall(): eng = MemoryEngine.create(":memory:") wid = eng.store.get_or_create_workspace("w") stale = eng.remember( @@ -518,16 +541,20 @@ def test_archive_logs_index_cleanup_failure_without_leaking_exception_text( ) eng.store.conn.commit() - def fail_delete(ids): - raise RuntimeError("credential-like index detail") - - monkeypatch.setattr(eng.index, "delete", fail_delete) - with caplog.at_level("WARNING", logger="engraphis.core.consolidate"): - report = consolidate(eng, workspace_id=wid) + archived_at = time.time() + report = consolidate(eng, workspace_id=wid, now=archived_at) assert [row["id"] for row in report["archived"]] == [stale] - assert "RuntimeError" in caplog.text - assert "credential-like index detail" not in caplog.text + assert eng.store.conn.execute( + "SELECT 1 FROM mem_vectors WHERE id=?", (stale,) + ).fetchone() is not None + valid_from = eng.store.get_memory(stale).valid_from + historical = eng.recall_engine.recall( + "What scratch note came from the old session?", + SearchFilter(workspace_id=wid, as_of=(valid_from + archived_at) / 2), + reinforce=False, + ) + assert [chunk["id"] for chunk in historical.chunks] == [stale] # ── explicit local consolidation command ───────────────────────────────────── diff --git a/tests/test_context_packing.py b/tests/test_context_packing.py new file mode 100644 index 00000000..7db323ce --- /dev/null +++ b/tests/test_context_packing.py @@ -0,0 +1,285 @@ +"""Focused contracts for deterministic, budgeted context packing.""" + +from __future__ import annotations + +import pytest +from typing import Optional + +from engraphis.core.context import DeterministicContextPacker, RegexTokenCounter +from engraphis.core.interfaces import Candidate, MemoryRecord + + +def _candidate( + memory_id: str, + content: str, + *, + score: float = 1.0, + arm: str = "semantic", + title: str = "Deployment", + summary: str = "", + metadata: Optional[dict[str, object]] = None, +) -> Candidate: + return Candidate( + id=memory_id, + score=score, + arm=arm, + record=MemoryRecord( + id=memory_id, + title=title, + content=content, + summary=summary, + repo_id="repo_demo", + metadata=metadata or {}, + ), + ) + + +def test_strict_budget_holds_when_the_first_source_is_oversized() -> None: + packer = DeterministicContextPacker() + candidate = _candidate( + "mem_oversized", + "Deploys must not run until backup passes. " + "This supporting explanation is deliberately much longer than the available budget. " + * 6, + ) + + context, chunks, usage = packer.pack("deploy backup", [candidate], token_budget=25) + + assert chunks + assert chunks[0].truncated is True + assert usage.context_tokens == RegexTokenCounter()(context) + assert usage.context_tokens <= usage.budget_tokens == 25 + + +def test_unfit_header_does_not_block_a_later_compact_source() -> None: + packer = DeterministicContextPacker() + candidates = [ + _candidate( + "mem_long_header", + "Alpha.", + score=1.0, + title="A title whose many separate words consume the entire tiny token budget", + ), + _candidate( + "mem_compact", + "Beta.", + score=0.9, + title="", + ), + ] + + context, chunks, usage = packer.pack("evidence", candidates, token_budget=6) + + assert [chunk.id for chunk in chunks] == ["mem_compact"] + assert "Beta." in context + assert usage.context_tokens <= 6 + + +def test_sentence_excerpt_marks_omission_and_preserves_qualifying_evidence() -> None: + packer = DeterministicContextPacker() + candidate = _candidate( + "mem_qualifier", + "Deploys are normally routine. " + "Deploys must not run until backup passes. " + "This unrelated operational history is only background. " + "More unrelated detail follows here.", + summary="Deploys run after backup.", + ) + + _, chunks, _ = packer.pack("deploy backup", [candidate], token_budget=25) + + assert len(chunks) == 1 + assert chunks[0].reason == "relevant_sentence_excerpt" + assert "must not run until backup passes" in chunks[0].excerpt.casefold() + assert "[…]" in chunks[0].excerpt + + +def test_qualifier_preserving_summary_is_preferred_when_it_fits() -> None: + packer = DeterministicContextPacker() + summary = "Deploys must not run until backup passes." + candidate = _candidate( + "mem_summary", + ("Deploys must not run until backup passes. " * 10).strip(), + summary=summary, + ) + + _, chunks, _ = packer.pack("deploy backup", [candidate], token_budget=30) + + assert len(chunks) == 1 + assert chunks[0].reason == "summary" + assert chunks[0].excerpt == summary + assert chunks[0].truncated is True + + +def test_summary_must_preserve_every_qualifier_before_it_can_replace_source() -> None: + packer = DeterministicContextPacker() + candidate = _candidate( + "mem_partial_qualifier", + "Deploys must run only when backup passes unless incident command grants an exception.", + summary="Deploys must run when backup passes.", + ) + + _, chunks, _ = packer.pack("deploy backup", [candidate], token_budget=40) + + assert len(chunks) == 1 + assert chunks[0].reason == "full" + assert "only when backup passes unless" in chunks[0].excerpt.casefold() + + +def test_tight_excerpt_prefers_a_separate_qualifier_sentence() -> None: + packer = DeterministicContextPacker() + candidate = _candidate( + "mem_separate_qualifier", + "Deploys run after backup. " + "Unless incident command approves an exception, deploys must not run.", + title="", + ) + + _, chunks, _ = packer.pack("deploy backup", [candidate], token_budget=18) + + assert len(chunks) == 1 + assert "unless" in chunks[0].excerpt.casefold() + assert "must not run" in chunks[0].excerpt.casefold() + + +def test_tight_budget_omits_evidence_if_a_late_qualifier_cannot_survive() -> None: + packer = DeterministicContextPacker() + candidate = _candidate( + "mem_late_qualifier", + "Use the fast deployment path in all environments except production.", + title="", + ) + + context, chunks, usage = packer.pack( + "deployment path", + [candidate], + token_budget=8, + ) + + assert context == "" + assert chunks == [] + assert usage.packed_count == 0 + + +@pytest.mark.parametrize("qualifier", ["without production approval", "but cannot run in production"]) +def test_tight_budget_does_not_strip_other_negative_qualifiers(qualifier: str) -> None: + packer = DeterministicContextPacker() + candidate = _candidate( + "mem_negative_qualifier", + f"Use the fast deployment path {qualifier}.", + title="", + ) + + context, chunks, usage = packer.pack( + "deployment path", + [candidate], + token_budget=8, + ) + + assert context == "" + assert chunks == [] + assert usage.packed_count == 0 + + +def test_custom_counter_can_truncate_inside_one_regex_token() -> None: + class CharacterCounter: + identity = "test.characters" + + def __call__(self, text: str) -> int: + return len(text) + + counter = CharacterCounter() + packer = DeterministicContextPacker(counter) + candidate = _candidate( + "mem_single_token", + "x" * 200, + title="", + ) + + context, chunks, usage = packer.pack( + "x", + [candidate], + token_budget=24, + ) + + assert chunks and chunks[0].truncated + assert usage.context_tokens == counter(context) + assert 0 < usage.context_tokens <= usage.budget_tokens == 24 + + +def test_supersession_and_claim_family_deduplication_keep_best_candidate() -> None: + packer = DeterministicContextPacker() + candidates = [ + _candidate( + "mem_original", + "The original rollout policy is recorded here.", + score=0.6, + metadata={"claim_key": "rollout-policy"}, + ), + _candidate( + "mem_revision", + "The revised rollout policy replaces the original.", + score=0.8, + metadata={"claim_key": "rollout-policy", "supersedes": "mem_original"}, + ), + _candidate( + "mem_latest", + "The current rollout policy is authoritative.", + score=0.9, + metadata={"supersedes": ["mem_revision"]}, + ), + ] + + _, chunks, usage = packer.pack("current rollout policy", candidates, token_budget=80) + + assert [chunk.id for chunk in chunks] == ["mem_latest"] + assert usage.packed_count == 1 + assert usage.omitted_count == 2 + + +@pytest.mark.parametrize("bridge_arm", ["graph", "code"]) +def test_graph_and_code_bridge_evidence_gets_selected_for_bridge_queries(bridge_arm: str) -> None: + packer = DeterministicContextPacker() + candidates = [ + _candidate( + "mem_vector", + "Dependency path summary is only generic information.", + score=0.60, + ), + _candidate( + "mem_bridge", + "Dependency path provides decisive evidence.", + score=0.50, + arm=bridge_arm, + ), + ] + + _, chunks, _ = packer.pack("why dependency path", candidates, token_budget=20) + + assert chunks[0].id == "mem_bridge" + assert chunks[0].reason == "bridge_evidence" + + +def test_packing_is_deterministic_and_reports_exact_usage_accounting() -> None: + packer = DeterministicContextPacker() + candidates = [ + _candidate("mem_a", "Alpha deployment evidence. Additional context follows."), + _candidate("mem_b", "Beta deployment evidence. Extra details follow.", score=0.9), + ] + + first = packer.pack("deployment evidence", candidates, token_budget=60) + second = packer.pack("deployment evidence", candidates, token_budget=60) + context, chunks, usage = first + + assert first == second + assert usage.context_tokens == RegexTokenCounter()(context) + assert usage.source_tokens == sum( + RegexTokenCounter()(f"{candidate.record.title}\n{candidate.record.content}") + for candidate in candidates + if candidate.record is not None + ) + assert usage.saved_tokens == max(0, usage.source_tokens - usage.context_tokens) + assert usage.savings_ratio == pytest.approx(usage.saved_tokens / usage.source_tokens) + assert usage.packed_count == len(chunks) + assert usage.omitted_count == len(candidates) - len(chunks) + assert usage.token_counter == "engraphis.regex.v1" diff --git a/tests/test_core_store.py b/tests/test_core_store.py index 4a3b4123..151dc671 100644 --- a/tests/test_core_store.py +++ b/tests/test_core_store.py @@ -22,7 +22,27 @@ def store(): def test_schema_version(store): - assert store.schema_version == 4 + assert store.schema_version == 5 + + +def test_clean_v5_schema_has_temporal_code_and_memory_link_tables(store): + tables = {row["name"] for row in store.conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall()} + link_columns = {row["name"] for row in store.conn.execute( + "PRAGMA table_info(code_memory_links)" + ).fetchall()} + incidence_columns = {row["name"] for row in store.conn.execute( + "PRAGMA table_info(memory_entities)" + ).fetchall()} + direct_link_columns = {row["name"] for row in store.conn.execute( + "PRAGMA table_info(mem_links)" + ).fetchall()} + + assert "memory_entities" in tables + assert {"valid_from", "valid_to", "ingested_at", "expired_at"} <= link_columns + assert {"memory_id", "entity_id", "source_kind", "confidence"} <= incidence_columns + assert {"valid_from", "valid_to", "valid_to_recorded_at", "ingested_at", "expired_at"} <= direct_link_columns def test_entity_normalization_preserves_meaningful_punctuation(): @@ -149,7 +169,7 @@ def test_v3_migration_classifies_existing_graph_layers_once(tmp_path): row = migrated.conn.execute( "SELECT layer FROM edges WHERE id='edge_old'" ).fetchone() - assert migrated.schema_version == 4 + assert migrated.schema_version == 5 assert row["layer"] == "entity" migrated.conn.execute( "UPDATE edges SET layer='causal' WHERE id='edge_old'" @@ -255,6 +275,162 @@ def test_graph_neighbors_filters_by_layer(store): assert store.neighbors(["deploy"], layers=[GraphLayer.TEMPORAL]) == [] +def test_graph_neighbors_apply_workspace_and_repo_scope(store): + w1 = store.get_or_create_workspace("w1") + w2 = store.get_or_create_workspace("w2") + r1 = store.get_or_create_repo(w1, "r") + r2 = store.get_or_create_repo(w2, "r") + store.upsert_edge(Edge( + id="", src="shared", dst="visible", relation="uses", + workspace_id=w1, repo_id=r1, + )) + store.upsert_edge(Edge( + id="", src="shared", dst="leaked", relation="uses", + workspace_id=w2, repo_id=r2, + )) + + rows = store.neighbors( + ["shared"], + flt=SearchFilter(workspace_id=w1, repo_id=r1), + ) + + assert [(edge.src, edge.dst) for edge in rows] == [("shared", "visible")] + + +def test_memory_links_honor_known_at_empty_layers_and_large_id_sets( + store, monkeypatch): + from engraphis.core import store as store_mod + + ids = [f"mem_{index:04d}" for index in range(600)] + store.add_link(ids[0], ids[-1], relation="causes", layer=GraphLayer.CAUSAL) + store.conn.execute( + "UPDATE mem_links SET created_at=100, valid_from=100, ingested_at=100" + ) + store.conn.commit() + monkeypatch.setattr(store_mod, "IN_CLAUSE_CHUNK", 50) + + assert store.links_among( + ids, flt=SearchFilter(known_at=99.0) + ) == [] + visible = store.links_among( + ids, flt=SearchFilter(known_at=100.0) + ) + assert [(row["a"], row["b"]) for row in visible] == [(ids[0], ids[-1])] + assert store.links_among(ids, layers=[]) == [] + + +def test_closed_memory_link_can_be_reactivated_without_erasing_history(store): + store.add_link( + "mem_a", "mem_b", relation="related", + valid_from=10.0, valid_to=20.0, valid_to_recorded_at=20.0, + ingested_at=10.0, + ) + assert not store.has_link("mem_a", "mem_b", relation="related") + + store.add_link( + "mem_b", "mem_a", relation="related", + valid_from=40.0, ingested_at=40.0, + ) + # Replaying the same current relation is still idempotent. + store.add_link( + "mem_a", "mem_b", relation="related", + valid_from=50.0, ingested_at=50.0, + ) + + rows = store.conn.execute( + "SELECT valid_from, valid_to, expired_at FROM mem_links " + "WHERE relation='related' ORDER BY valid_from" + ).fetchall() + assert [(row["valid_from"], row["valid_to"]) for row in rows] == [ + (10.0, 20.0), (40.0, None), + ] + assert store.has_link("mem_a", "mem_b", relation="related") + historical = store.links_among( + ["mem_a", "mem_b"], + flt=SearchFilter(valid_at=15.0, known_at=50.0), + ) + current = store.links_among( + ["mem_a", "mem_b"], + flt=SearchFilter(valid_at=50.0, known_at=50.0), + ) + assert [row["valid_from"] for row in historical] == [10.0] + assert [row["valid_from"] for row in current] == [40.0] + + +def test_expired_memory_link_does_not_block_reactivation(store): + store.add_link( + "mem_a", "mem_b", relation="related", + valid_from=10.0, ingested_at=10.0, expired_at=20.0, + ) + assert not store.has_link("mem_a", "mem_b", relation="related") + + store.add_link( + "mem_a", "mem_b", relation="related", + valid_from=40.0, ingested_at=40.0, + ) + + rows = store.conn.execute( + "SELECT valid_from, expired_at FROM mem_links ORDER BY valid_from" + ).fetchall() + assert [(row["valid_from"], row["expired_at"]) for row in rows] == [ + (10.0, 20.0), (40.0, None), + ] + assert [row["valid_from"] for row in store.links_among( + ["mem_a", "mem_b"], + flt=SearchFilter(valid_at=15.0, known_at=15.0), + )] == [10.0] + assert [row["valid_from"] for row in store.links_among( + ["mem_a", "mem_b"], + flt=SearchFilter(valid_at=50.0, known_at=50.0), + )] == [40.0] + + +def test_memory_link_metadata_change_versions_system_time_without_rewriting_history( + store, monkeypatch): + from engraphis.core import store as store_mod + + store.add_link( + "mem_a", "mem_b", relation="related", layer=GraphLayer.SEMANTIC, + reason="old evidence", valid_from=10.0, ingested_at=10.0, + ) + monkeypatch.setattr(store_mod, "now_ts", lambda: 30.0) + + store.add_link( + "mem_b", "mem_a", relation="related", layer=GraphLayer.CAUSAL, + reason="new evidence", + ) + # Replaying the converged metadata is idempotent, not another history row. + store.add_link( + "mem_a", "mem_b", relation="related", layer=GraphLayer.CAUSAL, + reason="new evidence", + ) + + rows = store.conn.execute( + "SELECT layer, reason, valid_from, ingested_at, expired_at " + "FROM mem_links ORDER BY ingested_at" + ).fetchall() + assert [tuple(row) for row in rows] == [ + ("semantic", "old evidence", 10.0, 10.0, 30.0), + ("causal", "new evidence", 10.0, 30.0, None), + ] + + past = SearchFilter(valid_at=20.0, known_at=20.0) + current = SearchFilter(valid_at=20.0, known_at=30.0) + assert [(row["layer"], row["reason"]) for row in store.get_links( + "mem_a", flt=past, + )] == [("semantic", "old evidence")] + assert [(row["layer"], row["reason"]) for row in store.get_links( + "mem_a", flt=current, + )] == [("causal", "new evidence")] + assert store.links_among( + ["mem_a", "mem_b"], layers=[GraphLayer.CAUSAL], flt=past, + ) == [] + assert store.links_among( + ["mem_a", "mem_b"], layers=[GraphLayer.SEMANTIC], flt=current, + ) == [] + assert store.has_link("mem_a", "mem_b", relation="related") + + def test_code_listing_helpers_honor_limit(store): """service.graph() bounds its per-repo code fetches; the SQL layer must actually enforce the cap rather than materializing the whole repo.""" @@ -310,6 +486,25 @@ def test_reinforce_increases_stability_and_count(store): assert after.stability > before.stability +def test_zero_temporal_anchors_round_trip_without_becoming_present_time(store): + wid = store.get_or_create_workspace("w") + mid = store.add_memory(MemoryRecord( + id="", content="known at the epoch", workspace_id=wid, + valid_from=0.0, ingested_at=0.0, last_access=0.0, + )) + edge_id = store.upsert_edge(Edge( + id="", src="a", dst="b", relation="uses", workspace_id=wid, + valid_from=0.0, ingested_at=0.0, + )) + + record = store.get_memory(mid) + edge = store.conn.execute( + "SELECT valid_from, ingested_at FROM edges WHERE id=?", (edge_id,) + ).fetchone() + assert record.valid_from == record.ingested_at == record.last_access == 0.0 + assert edge["valid_from"] == edge["ingested_at"] == 0.0 + + def test_symbol_roundtrip_and_search(store): sid = store.upsert_symbol(repo_id="repo_x", kind="function", name="add", fqname="add", file="calc.py", span="1-2", signature="def add(a, b):", @@ -330,6 +525,103 @@ def test_clear_symbols_for_file_replaces_not_accumulates(store): assert names == {"new"} +def test_code_history_closes_live_rows_and_supports_time_travel(store): + """Re-indexing retires old code evidence without deleting its history.""" + wid = store.get_or_create_workspace("w") + rid = store.get_or_create_repo(wid, "r") + mid = store.add_memory(MemoryRecord( + id="", content="The old implementation called helper.", + workspace_id=wid, repo_id=rid, scope=Scope.REPO, + )) + symbol_id = store.upsert_symbol( + repo_id=rid, kind="function", name="old", fqname="old", + file="calc.py", span="1-1", + ) + store.add_code_edge(repo_id=rid, src="old", dst="helper", relation="calls", + file="calc.py", line=1) + store.link_memory_symbol(repo_id=rid, symbol_id=symbol_id, memory_id=mid) + # Use a stable world/system-time interval rather than relying on sub-millisecond + # spacing between indexing and retirement on fast CI hosts. + store.conn.execute( + "UPDATE symbols SET valid_from=10, ingested_at=10 WHERE id=?", (symbol_id,) + ) + store.conn.execute( + "UPDATE memories SET valid_from=10, ingested_at=10 WHERE id=?", (mid,) + ) + store.conn.execute( + "UPDATE code_edges SET valid_from=10, ingested_at=10 WHERE repo_id=?", (rid,) + ) + store.conn.execute( + "UPDATE code_memory_links SET valid_from=10, ingested_at=10 WHERE repo_id=?", (rid,) + ) + store.conn.commit() + store.clear_symbols_for_file(rid, "calc.py") + + closed_at = store.conn.execute( + "SELECT valid_to FROM symbols WHERE id=?", (symbol_id,) + ).fetchone()["valid_to"] + assert store.list_symbols(rid) == [] + assert store.list_code_edges(rid) == [] + assert store.list_code_memory_links(rid) == [] + + history = SearchFilter(valid_at=11.0, + known_at=float(closed_at) + 1.0) + assert [row["id"] for row in store.list_symbols(rid, flt=history)] == [symbol_id] + assert [row["id"] for row in store.search_symbols(rid, "old", flt=history)] == [ + symbol_id + ] + assert len(store.list_code_edges(rid, flt=history)) == 1 + assert len(store.get_symbol_callers(rid, "helper", flt=history)) == 1 + assert len(store.list_code_memory_links(rid, flt=history)) == 1 + assert [row["id"] for row in store.symbols_for_memory( + rid, mid, flt=history + )] == [symbol_id] + + +def test_code_memory_link_listing_requires_visible_symbol_and_memory(store): + wid = store.get_or_create_workspace("w") + rid = store.get_or_create_repo(wid, "r") + mid = store.add_memory(MemoryRecord( + id="", content="deploy", workspace_id=wid, repo_id=rid, scope=Scope.REPO, + )) + symbol_id = store.upsert_symbol( + repo_id=rid, kind="function", name="deploy", fqname="deploy", + file="deploy.py", span="1-1", + ) + store.link_memory_symbol( + repo_id=rid, symbol_id=symbol_id, memory_id=mid, + ) + assert len(store.list_code_memory_links(rid)) == 1 + + # Simulate a legacy/direct writer that retired the symbol but forgot to + # retire its bridge. The read must still fail closed. + store.conn.execute( + "UPDATE symbols SET valid_to=0 WHERE id=?", (symbol_id,) + ) + store.conn.commit() + + assert store.list_code_memory_links(rid) == [] + + +def test_memory_entity_incidence_is_scoped_and_temporal(store): + wid = store.get_or_create_workspace("w") + rid = store.get_or_create_repo(wid, "r") + mid = store.add_memory(MemoryRecord( + id="", content="Alice owns the deployment.", workspace_id=wid, + repo_id=rid, scope=Scope.REPO, + )) + entity_id = store.upsert_entity(Node( + id="", name="Alice", ntype="person", workspace_id=wid, repo_id=rid, + )) + store.link_memory_entity( + memory_id=mid, entity_id=entity_id, workspace_id=wid, repo_id=rid, + source_kind="text_mention", confidence=0.8, + ) + rows = store.list_memory_entities(SearchFilter(workspace_id=wid, repo_id=rid)) + assert [(row["memory_id"], row["entity_id"], row["source_kind"]) + for row in rows] == [(mid, entity_id, "text_mention")] + + def test_explicit_semantic_code_edge_preserves_its_layer(store): store.add_code_edge( repo_id="repo_x", src="deploy", dst="release", relation="related_to", diff --git a/tests/test_engine.py b/tests/test_engine.py index 3a31280b..84c888f4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,6 +1,7 @@ import os import sqlite3 import tempfile +import time import pytest @@ -21,6 +22,26 @@ def test_engine_remember_and_recall(): assert "actions" in res.context.lower() or "aws" in res.context.lower() +def test_engine_recall_requires_explicit_reinforcement_signal(): + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "r") + mid = eng.remember("The deployment target is AWS ECS.", workspace_id=wid, repo_id=rid) + before = eng.store.get_memory(mid).access_count + + eng.recall("unrelated lunch menu", workspace_id=wid, repo_id=rid, k=1) + assert eng.store.get_memory(mid).access_count == before + + eng.recall( + "deployment target", + workspace_id=wid, + repo_id=rid, + k=1, + reinforce=True, + ) + assert eng.store.get_memory(mid).access_count > before + + def test_index_upsert_failure_preserves_memory_and_audits(caplog): class BrokenIndex: def search(self, _vec, _k, *, filter=None): @@ -302,6 +323,9 @@ def test_forget_invalidates_without_deleting(): eng.forget(mid, reason="no longer true") assert mid not in [m.id for m in eng.store.list_memories(SearchFilter(workspace_id=wid))] assert eng.store.get_memory(mid) is not None # not hard-deleted + assert eng.store.conn.execute( + "SELECT 1 FROM mem_vectors WHERE id=?", (mid,) + ).fetchone() is not None def test_forget_unknown_id_raises(): @@ -343,6 +367,9 @@ def test_correct_supersedes_without_deleting(): assert new_rec.metadata.get("corrects") == mid live_ids = [m.id for m in eng.store.list_memories(SearchFilter(workspace_id=wid))] assert mid not in live_ids and out["id"] in live_ids + assert eng.store.conn.execute( + "SELECT 1 FROM mem_vectors WHERE id=?", (mid,) + ).fetchone() is not None def test_promote_widens_scope_and_preserves_source_history_and_safety(): @@ -439,6 +466,74 @@ def test_timeline_orders_history_chronologically(): assert hist[0].valid_from < hist[1].valid_from +def test_temporal_supersession_closes_at_effective_time_and_keeps_vectors(): + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "r") + old = eng.remember( + "The API rate limit is 100 requests per minute.", + workspace_id=wid, + repo_id=rid, + valid_from=1_000.0, + ) + new = eng.remember( + "The API rate limit is 500 requests per minute.", + workspace_id=wid, + repo_id=rid, + valid_from=2_000.0, + ) + + assert eng.store.get_memory(old).valid_to == 2_000.0 + assert eng.store.conn.execute( + "SELECT 1 FROM mem_vectors WHERE id=?", (old,) + ).fetchone() is not None + before = eng.recall_engine.recall( + "What is the API rate limit?", + SearchFilter(workspace_id=wid, repo_id=rid, as_of=1_500.0), + reinforce=False, + ) + after = eng.recall_engine.recall( + "What is the API rate limit?", + SearchFilter(workspace_id=wid, repo_id=rid, as_of=2_500.0), + reinforce=False, + ) + assert [chunk["id"] for chunk in before.chunks] == [old] + assert [chunk["id"] for chunk in after.chunks] == [new] + + +@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), "not-a-time", True]) +def test_remember_rejects_non_finite_valid_from_without_writing(invalid): + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + + with pytest.raises(ValueError, match="valid_from must be a finite timestamp"): + eng.remember("A fact.", workspace_id=wid, valid_from=invalid) + + assert eng.store.list_memories(SearchFilter(workspace_id=wid)) == [] + + +def test_backdated_supersession_is_rejected_without_creating_an_invalid_interval(): + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + old = eng.remember( + "The deployment window is Friday afternoon.", + workspace_id=wid, + valid_from=2_000.0, + ) + + with pytest.raises(ValueError, match="cannot predate"): + eng.remember( + "The deployment window is Thursday afternoon.", + workspace_id=wid, + valid_from=1_000.0, + ) + + assert eng.store.get_memory(old).valid_to is None + assert len(eng.store.list_memories( + SearchFilter(workspace_id=wid), include_invalid=True + )) == 1 + + def test_recall_proactive_includes_last_session_handoff(): eng = MemoryEngine.create(":memory:") wid = eng.store.get_or_create_workspace("w") @@ -827,6 +922,66 @@ def test_code_memory_paths_hide_forgotten_memories(tmp_path): assert eng.analyze_impact(["deploy.py"], repo_id=rid)["memory_mentions"] == [] +def test_code_search_and_memory_paths_honor_historical_anchors(): + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "sample") + symbol_id = eng.store.upsert_symbol( + repo_id=rid, kind="function", name="old_fn", fqname="old_fn", + file="old.py", span="1-1", + ) + eng.store.add_code_edge( + repo_id=rid, src="caller", dst="old_fn", relation="calls", + file="old.py", line=2, + ) + memory_id = eng.store.add_memory(MemoryRecord( + id="", content="old_fn used the historical path", title="old path", + workspace_id=wid, repo_id=rid, scope=Scope.REPO, + valid_from=10.0, ingested_at=10.0, + )) + eng.store.link_memory_symbol( + repo_id=rid, symbol_id=symbol_id, memory_id=memory_id, + ) + for table in ("symbols", "code_edges", "code_memory_links"): + eng.store.conn.execute( + f"UPDATE {table} SET valid_from=10, ingested_at=10 WHERE repo_id=?", + (rid,), + ) + eng.store.conn.commit() + eng.store.close_validity(memory_id, at=20.0) + eng.store.clear_symbols_for_file(rid, "old.py") + symbol_closed_at = eng.store.conn.execute( + "SELECT valid_to FROM symbols WHERE id=?", (symbol_id,) + ).fetchone()["valid_to"] + historical = SearchFilter( + workspace_id=wid, + repo_id=rid, + valid_at=15.0, + known_at=float(symbol_closed_at) + 1.0, + ) + + search = eng.search_code("old_fn", repo_id=rid, flt=historical) + + assert [symbol["id"] for symbol in search["symbols"]] == [symbol_id] + assert search["symbols"][0]["called_by"][0]["src"] == "caller" + assert eng.code_path( + "old_fn", memory_id, repo_id=rid, flt=historical, + )["found"] is True + impact = eng.analyze_impact(["old.py"], repo_id=rid, flt=historical) + assert {row["id"] for row in impact["symbols"]} == {symbol_id} + assert {row["id"] for row in impact["memory_mentions"]} == {memory_id} + assert impact["graph"]["edges"] == 1 + exported = eng.export_code_graph(repo_id=rid, flt=historical) + assert {row["id"] for row in exported["nodes"]} == {symbol_id} + assert len(exported["edges"]) == 1 + assert {row["memory_id"] for row in exported["memory_links"]} == {memory_id} + assert eng.code_path("old_fn", memory_id, repo_id=rid)["found"] is False + assert eng.analyze_impact( + ["old.py"], repo_id=rid + )["memory_mentions"] == [] + assert eng.export_code_graph(repo_id=rid)["nodes"] == [] + + def test_code_reads_apply_session_visibility_to_every_memory_surface(): eng = MemoryEngine.create(":memory:") wid = eng.store.get_or_create_workspace("w") @@ -879,6 +1034,33 @@ def test_code_reads_apply_session_visibility_to_every_memory_surface(): )["found"] +def test_code_reads_reject_mismatched_workspace_or_repo_filters(): + eng = MemoryEngine.create(":memory:") + first_workspace = eng.store.get_or_create_workspace("first") + second_workspace = eng.store.get_or_create_workspace("second") + first_repo = eng.store.get_or_create_repo(first_workspace, "api") + second_repo = eng.store.get_or_create_repo(second_workspace, "api") + eng.store.upsert_symbol( + repo_id=second_repo, kind="function", name="secret_fn", + fqname="secret_fn", file="secret.py", span="1-1", + ) + + with pytest.raises(ValueError, match="workspace_id"): + eng.search_code( + "secret_fn", + repo_id=second_repo, + flt=SearchFilter(workspace_id=first_workspace, repo_id=second_repo), + ) + with pytest.raises(ValueError, match="repo_id"): + eng.export_code_graph( + repo_id=second_repo, + flt=SearchFilter( + workspace_id=second_workspace, + repo_id=first_repo, + ), + ) + + def test_rebuild_code_memory_links_keysets_past_five_thousand_session_records(): eng = MemoryEngine.create(":memory:") wid = eng.store.get_or_create_workspace("w") @@ -1126,3 +1308,54 @@ def test_code_matcher_cache_is_invalidated_when_symbols_change(): assert [r["symbol_id"] for r in eng.store.list_code_memory_links(rid) if r["memory_id"] == second], "a new symbol must invalidate the cached matcher" + + +def test_extracted_graph_evidence_inherits_memory_temporal_anchors(): + eng = MemoryEngine.create(":memory:", graph_extractor="regex") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "r") + future = time.time() + 10_000 + first = eng.remember( + "Alice uses Stripe.", + workspace_id=wid, + repo_id=rid, + valid_from=future, + resolve_conflicts=False, + ) + memory = eng.store.get_memory(first) + edges = eng.store.edges_in_scope(SearchFilter( + workspace_id=wid, + repo_id=rid, + valid_at=future, + known_at=memory.ingested_at, + )) + assert len(edges) == 1 + assert edges[0].valid_from == future + assert edges[0].ingested_at == memory.ingested_at + assert eng.store.edges_in_scope(SearchFilter( + workspace_id=wid, + repo_id=rid, + valid_at=future - 1, + known_at=memory.ingested_at, + )) == [] + assert eng.store.edges_in_scope(SearchFilter( + workspace_id=wid, + repo_id=rid, + valid_at=future, + known_at=memory.ingested_at - 1, + )) == [] + + earlier = future - 500 + eng.remember( + "Alice uses Stripe.", + workspace_id=wid, + repo_id=rid, + valid_from=earlier, + resolve_conflicts=False, + ) + edge = eng.store.edges_in_scope(SearchFilter( + workspace_id=wid, + repo_id=rid, + valid_at=earlier, + ))[0] + assert edge.valid_from == earlier diff --git a/tests/test_graphrank.py b/tests/test_graphrank.py index 704b8717..c1e7b841 100644 --- a/tests/test_graphrank.py +++ b/tests/test_graphrank.py @@ -1,7 +1,9 @@ import numpy as np # noqa: F401 (asserts numpy-only dependency stays importable) +import pytest from engraphis.backends import DeterministicEmbedder, NumpyVectorIndex from engraphis.backends.reranker import IdentityReranker +from engraphis.core import graphrank from engraphis.core.graphrank import personalized_pagerank from engraphis.core.interfaces import Edge, MemoryRecord, MemoryType, Node, Scope, SearchFilter from engraphis.core.recall import RecallEngine @@ -38,6 +40,70 @@ def test_ppr_weight_influences_ranking(): assert r["c"] > r["d"] +def _dense_reference(adjacency, seeds, *, damping=0.85, iterations=30, tol=1e-9): + """Pre-sparse implementation retained only as a numerical oracle for this test.""" + nodes = sorted(set(adjacency) | {dst for edges in adjacency.values() for dst, _ in edges} + | set(seeds)) + index = {node: position for position, node in enumerate(nodes)} + seed_ids = [index[seed] for seed in seeds if seed in index] + if not seed_ids or not [seed for seed in seeds if seed in adjacency and adjacency[seed]]: + return {} + matrix = np.zeros((len(nodes), len(nodes)), dtype=np.float64) + for source, edges in adjacency.items(): + total = float(sum(max(weight, 0.0) for _, weight in edges)) + if total <= 0.0: + continue + for destination, weight in edges: + if weight > 0.0: + matrix[index[destination], index[source]] += weight / total + restart = np.zeros(len(nodes), dtype=np.float64) + restart[seed_ids] = 1.0 / len(seed_ids) + dangling = matrix.sum(axis=0) == 0.0 + probability = restart.copy() + for _ in range(iterations): + next_probability = (1.0 - damping) * restart + damping * ( + matrix @ probability + probability[dangling].sum() * restart + ) + if float(np.abs(next_probability - probability).sum()) < tol: + probability = next_probability + break + probability = next_probability + return {nodes[index]: float(score) for index, score in enumerate(probability) if score > 0.0} + + +def test_sparse_ppr_matches_prior_dense_iteration_within_tight_tolerance(): + adj = { + "a": [("b", 2.0), ("b", 1.0), ("c", 0.5)], + "b": [("a", 1.0), ("d", 3.0)], + "c": [], + "d": [("a", 1.0)], + } + expected = _dense_reference(adj, ["a", "missing"], iterations=50) + actual = personalized_pagerank(adj, ["a", "missing"], iterations=50) + assert actual.keys() == expected.keys() + for node in actual: + assert actual[node] == pytest.approx(expected[node], abs=1e-12) + + +def test_sparse_ppr_handles_several_thousand_node_graph_without_dense_matrix(): + # A bidirectional chain has ~12k directed edges. A dense 6k × 6k float64 + # matrix would require ~275 MiB before iteration buffers; sparse iteration + # remains proportional to the chain itself. + size = 6_000 + adjacency = {f"n{index}": [] for index in range(size)} + for index in range(size - 1): + adjacency[f"n{index}"].append((f"n{index + 1}", 1.0)) + adjacency[f"n{index + 1}"].append((f"n{index}", 1.0)) + result = personalized_pagerank(adjacency, ["n0"]) + assert result["n0"] > result["n10"] > 0.0 + assert abs(sum(result.values()) - 1.0) < 1e-9 + + +def test_sparse_ppr_refuses_oversized_direct_input_deterministically(monkeypatch): + monkeypatch.setattr(graphrank, "MAX_NODES", 2) + assert personalized_pagerank({"a": [("b", 1.0)], "b": [("c", 1.0)]}, ["a"]) == {} + + # ── PPR retrieval arm inside RecallEngine ───────────────────────────────────────── def _graph_fixture(): @@ -66,6 +132,14 @@ def _graph_fixture(): rec = MemoryRecord(id="", content=text, mtype=MemoryType.SEMANTIC, scope=Scope.REPO, workspace_id=wid, repo_id=rid, embedding=emb.embed([text])[0]) ids[tag] = store.add_memory(rec) + entity = entity_ids["alphasvc" if tag == "m1" else "gammasvc"] + store.link_memory_entity( + memory_id=ids[tag], + entity_id=entity, + workspace_id=wid, + repo_id=rid, + source_kind="test", + ) return store, wid, emb, index, ids @@ -117,6 +191,10 @@ def test_1hop_arm_honors_graph_layer_filter(): id="", content=text, mtype=MemoryType.SEMANTIC, scope=Scope.WORKSPACE, workspace_id=wid, embedding=emb.embed([text])[0], )) + store.link_memory_entity( + memory_id=mid, entity_id=b, workspace_id=wid, repo_id=None, + source_kind="test", + ) eng = RecallEngine(store, emb, index, IdentityReranker(), graph_mode="1hop") now = now_ts() @@ -131,6 +209,63 @@ def test_1hop_arm_honors_graph_layer_filter(): store.close() +def test_ppr_mem_links_obey_known_at_and_empty_layer_filter(): + store = Store(":memory:") + wid = store.get_or_create_workspace("w") + emb = DeterministicEmbedder(dim=64) + index = NumpyVectorIndex(store) + alpha = store.upsert_entity(Node( + id="", name="alphasvc", ntype="service", workspace_id=wid + )) + beta = store.upsert_entity(Node( + id="", name="betasvc", ntype="service", workspace_id=wid + )) + first = store.add_memory(MemoryRecord( + id="", content="alphasvc entry point", workspace_id=wid, + scope=Scope.WORKSPACE, valid_from=10.0, ingested_at=10.0, + )) + second = store.add_memory(MemoryRecord( + id="", content="betasvc ledger", workspace_id=wid, + scope=Scope.WORKSPACE, valid_from=10.0, ingested_at=10.0, + )) + store.link_memory_entity( + memory_id=first, entity_id=alpha, workspace_id=wid, repo_id=None, + valid_from=10.0, ingested_at=10.0, + ) + store.link_memory_entity( + memory_id=second, entity_id=beta, workspace_id=wid, repo_id=None, + valid_from=10.0, ingested_at=10.0, + ) + store.add_link(first, second) + store.conn.execute( + "UPDATE mem_links SET created_at=100, valid_from=100, ingested_at=100 " + "WHERE a=? AND b=?", (first, second) + ) + store.conn.commit() + engine = RecallEngine(store, emb, index, IdentityReranker()) + + before = engine._graph_arm( + "alphasvc", + SearchFilter(workspace_id=wid, valid_at=50.0, known_at=50.0), + 50.0, + ) + after = engine._graph_arm( + "alphasvc", + SearchFilter(workspace_id=wid, valid_at=150.0, known_at=150.0), + 150.0, + ) + disabled = engine._graph_arm( + "alphasvc", + SearchFilter(workspace_id=wid, graph_layers=[]), + 150.0, + ) + + assert first in before and second not in before + assert second in after + assert disabled == {} + store.close() + + def test_recall_end_to_end_with_ppr_default(): store, wid, emb, index, ids = _graph_fixture() eng = RecallEngine(store, emb, index, IdentityReranker()) diff --git a/tests/test_grounded.py b/tests/test_grounded.py index 1017d8b4..c48c2355 100644 --- a/tests/test_grounded.py +++ b/tests/test_grounded.py @@ -51,6 +51,24 @@ def test_grounded_abstains_off_topic(): assert ans.reason +def test_grounded_abstains_when_distractor_shares_only_a_topic_keyword(): + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "r") + eng.remember( + "The office kitchen orders sourdough every Friday.", + workspace_id=wid, + repo_id=rid, + ) + + ans = eng.grounded_recall( + "How do I bake sourdough bread?", workspace_id=wid, repo_id=rid, + ) + + assert ans.abstained and not ans.grounded + assert ans.support < GROUNDED_SUPPORT_FLOOR + + def test_grounded_abstains_on_empty_store(): eng = MemoryEngine.create(":memory:") wid = eng.store.get_or_create_workspace("w") @@ -145,6 +163,7 @@ def test_service_grounded_recall_shape(): svc.remember("We use PASETO for auth.", workspace="acme", repo="backend", title="auth") out = svc.grounded_recall("which auth scheme did we standardise on?", workspace="acme", repo="backend") assert {"query", "grounded", "abstained", "answer", "support", "citations"} <= set(out) + assert out["receipt"]["operation"] == "grounded_recall" def test_service_grounded_recall_unknown_workspace_is_soft(): @@ -258,6 +277,108 @@ def test_llm_prose_without_citation_falls_back_to_extractive(): assert "kerberos" not in ans.answer.lower() # uncited prose rejected +def test_llm_prose_with_any_out_of_range_citation_falls_back_to_extractive(): + eng, wid, rid = _engine_with_facts() + ans = eng.grounded_recall( + "which auth scheme did we standardise on?", + workspace_id=wid, + repo_id=rid, + llm=_FakeLLM("PASETO is supported by [1], while Kerberos is supported by [99]."), + ) + assert ans.grounded and ans.synthesized is False + assert "kerberos" not in ans.answer.lower() + + +def test_llm_invented_fact_with_valid_marker_falls_back_to_extractive(): + # A valid [1] marker alone is not evidence for the generated claim. + eng, wid, rid = _engine_with_facts() + ans = eng.grounded_recall( + "which auth scheme did we standardise on?", + workspace_id=wid, + repo_id=rid, + llm=_FakeLLM("Invented fact [1]."), + ) + assert ans.grounded and ans.synthesized is False + assert "invented fact" not in ans.answer.lower() + assert "paseto" in ans.answer.lower() + + +def test_llm_reordered_source_tokens_cannot_reverse_the_grounded_claim(): + eng, wid, rid = _engine_with_facts() + generated = "We standardised on JWT tokens for auth, replacing PASETO [1]." + + ans = eng.grounded_recall( + "which auth scheme did we standardise on?", + workspace_id=wid, + repo_id=rid, + llm=_FakeLLM(generated), + ) + + assert ans.grounded and ans.synthesized is False + assert ans.answer != generated + assert "paseto tokens for auth, replacing jwt" in ans.answer.lower() + + +def test_llm_uncited_second_sentence_falls_back_even_when_its_words_are_in_source(): + eng, wid, rid = _engine_with_facts() + generated = "PASETO, per source [1]. JWT tokens." + ans = eng.grounded_recall( + "which auth scheme did we standardise on?", + workspace_id=wid, + repo_id=rid, + llm=_FakeLLM(generated), + ) + assert ans.grounded and ans.synthesized is False + assert ans.answer != generated + + +def test_tiny_budget_cannot_ground_from_raw_unpacked_memory(): + eng = MemoryEngine.create(":memory:") + wid = eng.store.get_or_create_workspace("w") + rid = eng.store.get_or_create_repo(wid, "r") + eng.remember( + "PASETO authenticates API requests. " + "Unpacked private detail. " * 500, + workspace_id=wid, + repo_id=rid, + ) + + ans = eng.grounded_recall( + "How are API requests authenticated?", + workspace_id=wid, + repo_id=rid, + min_support=0.0, + token_budget=1, + ) + + assert ans.abstained and not ans.grounded + assert ans.answer == "" and ans.citations == [] + assert ans.packed_sources == [] + assert ans.usage["answer_tokens"] == 0 + + +@pytest.mark.parametrize("min_support", [float("nan"), -0.1, 1.1]) +def test_grounded_recall_rejects_invalid_support_thresholds(min_support): + eng, wid, rid = _engine_with_facts() + with pytest.raises(ValueError, match="min_support"): + eng.grounded_recall( + "which auth scheme did we standardise on?", + workspace_id=wid, + repo_id=rid, + min_support=min_support, + ) + + +def test_grounded_recall_rejects_zero_citation_budget(): + eng, wid, rid = _engine_with_facts() + with pytest.raises(ValueError, match="max_citations"): + eng.grounded_recall( + "which auth scheme did we standardise on?", + workspace_id=wid, + repo_id=rid, + max_citations=0, + ) + + def test_llm_abstain_sentinel_has_no_citations(): # S1: an abstain (either path) carries no citations, for contract parity. eng, wid, rid = _engine_with_facts() diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 205a9edb..8e623f51 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -53,7 +53,8 @@ def _recall_side_effect_snapshot(srv): _ALL_TOOLS = { - "engraphis_remember", "engraphis_recall", "engraphis_why", "engraphis_timeline", + "engraphis_remember", "engraphis_recall", "engraphis_recall_context", + "engraphis_why", "engraphis_timeline", "engraphis_recall_proactive", "engraphis_forget", "engraphis_pin", "engraphis_correct", "engraphis_promote", "engraphis_link", "engraphis_record_event", "engraphis_index_repo", "engraphis_search_code", "engraphis_code_path", "engraphis_code_impact", @@ -78,14 +79,30 @@ def test_server_identity_and_tools_registered(): assert "engraphis_end_session" in srv.mcp.instructions assert "open_threads=[]" in srv.mcp.instructions tools = {t.name: t for t in asyncio.run(srv.mcp.list_tools())} - assert len(_ALL_TOOLS) == 29 + assert len(_ALL_TOOLS) == 30 assert set(tools) == _ALL_TOOLS kilo = (ROOT / "docs" / "KILO_CODE_INTEGRATION.md").read_text(encoding="utf-8") - full_surface = kilo.split("## 4. The 29 tools", 1)[1].split("\n---", 1)[0] + full_surface = kilo.split("## 4. The 30 tools", 1)[1].split("\n---", 1)[0] assert set(re.findall(r"`(engraphis_[a-z_]+)`", full_surface)) == _ALL_TOOLS # Flat schema (not a nested "params" object) so agents can call fields directly. props = tools["engraphis_remember"].inputSchema.get("properties", {}) assert "content" in props and "workspace" in props and "params" not in props + assert {"valid_from", "subject_key", "claim_kind"} <= set(props) + assert "as_of" in tools["engraphis_recall"].inputSchema.get("properties", {}) + assert {"valid_at", "known_at", "token_budget", "retrieval_profile", + "response_mode", "diagnostics"} <= set( + tools["engraphis_recall"].inputSchema.get("properties", {}) + ) + assert tools["engraphis_recall_context"].inputSchema["properties"][ + "token_budget" + ]["default"] == 1024 + assert "as_of" in tools["engraphis_recall_grounded"].inputSchema.get("properties", {}) + assert {"valid_at", "known_at", "token_budget", "retrieval_profile", "response_mode"} <= set( + tools["engraphis_answer"].inputSchema.get("properties", {}) + ) + assert {"as_of", "valid_at", "known_at"} <= set( + tools["engraphis_export_code_graph"].inputSchema.get("properties", {}) + ) def test_mcp_server_module_entrypoint_runs_stdio_handshake(): @@ -122,7 +139,13 @@ def test_mcp_server_module_entrypoint_runs_stdio_handshake(): ( "engraphis_recall", {"query": "Which tokens authenticate the API?", "workspace": "acme", "repo": "api"}, + False, True, + ), + ( + "engraphis_recall_context", + {"query": "Which tokens authenticate the API?", "workspace": "acme", "repo": "api"}, + False, True, ), ( @@ -214,6 +237,74 @@ def test_remember_and_recall_tool_callables(monkeypatch): assert "GitHub Actions" in rec["context"] +def test_recall_context_returns_compact_sources_and_strict_usage(monkeypatch): + srv = _module_with_memory_db(monkeypatch) + json.loads(srv.engraphis_remember( + content=("Deploy via signed tags after backup verification. " * 20), + workspace="acme", + repo="infra", + )) + + recalled = json.loads(srv.engraphis_recall_context( + query="how do we deploy?", + workspace="acme", + repo="infra", + token_budget=48, + )) + + assert recalled["usage"]["context_tokens"] <= 48 + assert recalled["usage"]["token_counter"] == "engraphis.regex.v1" + assert recalled["sources"] + assert all("content" not in source for source in recalled["sources"]) + assert "memories" not in recalled + + +def test_recall_context_payload_saves_at_least_half_vs_full_recall(monkeypatch): + from engraphis.core.context import RegexTokenCounter + + srv = _module_with_memory_db(monkeypatch) + detail = ( + "The decision record includes migration notes, version constraints, rollback " + "steps, historical exceptions, and audit evidence retained for operators. " + ) + facts = ( + "We standardized on pnpm across frontend repositories. " + detail * 24, + "Backend dependency management uses Poetry. " + detail * 24, + "Design mockups and handoff use Figma. " + detail * 24, + "Continuous integration runs on GitHub Actions. " + detail * 24, + ) + for fact in facts: + json.loads(srv.engraphis_remember( + content=fact, workspace="acme", repo="platform", dedupe=False + )) + + full = srv.engraphis_recall( + query="What package manager do frontend repositories use?", + workspace="acme", + repo="platform", + k=4, + token_budget=96, + ) + compact = srv.engraphis_recall_context( + query="What package manager do frontend repositories use?", + workspace="acme", + repo="platform", + k=4, + token_budget=96, + ) + counter = RegexTokenCounter() + full_payload = json.loads(full) + compact_payload = json.loads(compact) + full_tokens = counter(full) + compact_tokens = counter(compact) + ratio = compact_tokens / full_tokens + + assert [source["id"] for source in compact_payload["sources"]] == [ + source["id"] for source in full_payload["packed_sources"] + ] + assert ratio <= 0.5, f"compact/full fixture ratio was {ratio:.4f}" + + def test_remember_reports_resolution_op(monkeypatch): srv = _module_with_memory_db(monkeypatch) text = "We standardized on pnpm as the package manager for all frontend repos." @@ -258,6 +349,69 @@ def test_grounded_recall_tool_returns_flat_answer_payload(monkeypatch): assert "PASETO" in alias["answer"] +def test_grounded_tool_positional_compatibility_keeps_support_and_synthesis_slots(monkeypatch): + """New temporal/packing fields must not reinterpret legacy direct Python calls.""" + srv = _module_with_memory_db(monkeypatch) + srv.engraphis_remember( + content="The API uses PASETO tokens for authentication.", + workspace="acme", repo="api", + ) + + # The final two positional arguments were min_support and synthesize in the + # published 1.x callable. A temporal field inserted before them would turn + # 0.0 into as_of and silently change the answer. + direct = json.loads(srv.engraphis_recall_grounded( + "Which auth tokens does the API use?", "acme", "api", None, None, + 8, 0.0, False, + )) + alias = json.loads(srv.engraphis_answer( + "Which auth tokens does the API use?", "acme", "api", 8, 0.0, False, + )) + + assert direct["grounded"] is True + assert alias["grounded"] is True + + +def test_mcp_tools_expose_point_in_time_write_and_recall(monkeypatch): + srv = _module_with_memory_db(monkeypatch) + old = json.loads(srv.engraphis_remember( + content="The API rate limit is 100 requests per minute.", + workspace="acme", + repo="api", + valid_from=1_000.0, + )) + new = json.loads(srv.engraphis_remember( + content="The API rate limit is 500 requests per minute.", + workspace="acme", + repo="api", + valid_from=2_000.0, + )) + + before = json.loads(srv.engraphis_recall( + query="What is the API rate limit?", + workspace="acme", + repo="api", + as_of=1_500.0, + )) + after = json.loads(srv.engraphis_recall_grounded( + query="What is the API rate limit?", + workspace="acme", + repo="api", + as_of=2_500.0, + min_support=0.0, + )) + alias = json.loads(srv.engraphis_answer( + query="What is the API rate limit?", + workspace="acme", + repo="api", + as_of=1_500.0, + min_support=0.0, + )) + assert [memory["id"] for memory in before["memories"]] == [old["id"]] + assert [citation["id"] for citation in after["citations"]] == [new["id"]] + assert [citation["id"] for citation in alias["citations"]] == [old["id"]] + + def test_tool_returns_actionable_error_on_bad_input(monkeypatch): srv = _module_with_memory_db(monkeypatch) out = srv.engraphis_remember(content="", workspace="acme") # empty content -> service rejects diff --git a/tests/test_read_only_api.py b/tests/test_read_only_api.py index a66b0390..633a7793 100644 --- a/tests/test_read_only_api.py +++ b/tests/test_read_only_api.py @@ -57,6 +57,56 @@ def test_read_only_api_serves_graph_and_intent_recall(): assert response.json()["operation"] == "recall" +def test_read_only_code_search_forwards_bitemporal_anchors(): + svc = MemoryService.create(":memory:", graph_extractor="none") + svc.remember("Code search anchor.", workspace="w", repo="repo") + observed = {} + original = svc.search_code + + def observe(*args, **kwargs): + observed.update(kwargs) + return original(*args, **kwargs) + + svc.search_code = observe + response = TestClient(create_read_only_app(svc)).get( + "/code/search", + params={ + "query": "missing", "workspace": "w", "repo": "repo", + "as_of": 10.0, "valid_at": 10.0, "known_at": 20.0, + }, + ) + + assert response.status_code == 200 + assert observed["as_of"] == observed["valid_at"] == 10.0 + assert observed["known_at"] == 20.0 + + +@pytest.mark.parametrize("path", ["/graph", "/code/export"]) +def test_read_only_graph_surfaces_forward_bitemporal_anchors(path): + svc = MemoryService.create(":memory:", graph_extractor="none") + svc.remember("Temporal adapter anchor.", workspace="w", repo="repo") + observed = {} + method_name = "graph" if path == "/graph" else "export_code_graph" + original = getattr(svc, method_name) + + def observe(*args, **kwargs): + observed.update(kwargs) + return original(*args, **kwargs) + + setattr(svc, method_name, observe) + response = TestClient(create_read_only_app(svc)).get( + path, + params={ + "workspace": "w", "repo": "repo", + "as_of": 10.0, "valid_at": 10.0, "known_at": 20.0, + }, + ) + + assert response.status_code == 200 + assert observed["as_of"] == observed["valid_at"] == 10.0 + assert observed["known_at"] == 20.0 + + def test_read_only_graph_does_not_lazy_backfill(): svc = MemoryService.create(":memory:", graph_extractor="none") svc.remember( diff --git a/tests/test_recall.py b/tests/test_recall.py index 9f82483c..e674a1a2 100644 --- a/tests/test_recall.py +++ b/tests/test_recall.py @@ -51,13 +51,28 @@ def test_recall_bitemporal_excludes_invalidated_fact(): assert old not in [c["id"] for c in res.chunks] -def test_recall_reinforces_returned_memories(): +def test_recall_is_observational_by_default(): store, emb, eng = _engine() wid = store.get_or_create_workspace("w") rid = store.get_or_create_repo(wid, "r") mid = _add(store, emb, wid, rid, "pnpm is our package manager.") before = store.get_memory(mid).access_count eng.recall("package manager", SearchFilter(workspace_id=wid), k=1) + assert store.get_memory(mid).access_count == before + + +def test_recall_can_reinforce_when_use_is_explicit(): + store, emb, eng = _engine() + wid = store.get_or_create_workspace("w") + rid = store.get_or_create_repo(wid, "r") + mid = _add(store, emb, wid, rid, "pnpm is our package manager.") + before = store.get_memory(mid).access_count + eng.recall( + "package manager", + SearchFilter(workspace_id=wid), + k=1, + reinforce=True, + ) assert store.get_memory(mid).access_count > before diff --git a/tests/test_receipts.py b/tests/test_receipts.py index abea05ef..c692416e 100644 --- a/tests/test_receipts.py +++ b/tests/test_receipts.py @@ -1,6 +1,11 @@ +import hashlib import json +import sqlite3 from concurrent.futures import ThreadPoolExecutor +import pytest + +from engraphis.core.ids import new_id from engraphis.core.store import Store from engraphis.service import MemoryService @@ -41,6 +46,89 @@ def test_receipts_are_content_free_and_tamper_evident(): } +def test_short_user_controlled_receipt_labels_are_never_stored_verbatim(): + store = Store(":memory:") + wid = store.get_or_create_workspace("w") + secret = "sk-live-short" + receipt = store.record_receipt( + "recall", + workspace_id=wid, + metadata={ + "intent": secret, + "relation": secret, + "token_usage": {"context_tokens": 1, "token_counter": secret}, + }, + ) + + encoded = json.dumps(receipt) + assert secret not in encoded + assert receipt["metadata"]["intent"].startswith("sha256:") + assert receipt["metadata"]["relation"].startswith("sha256:") + assert receipt["metadata"]["token_usage"]["token_counter"].startswith("sha256:") + + +def test_store_list_receipts_never_reflects_poisoned_storage_fields(): + store = Store(":memory:") + wid = store.get_or_create_workspace("w") + receipt = store.record_receipt("recall", workspace_id=wid) + markers = { + "id": "STORE_LIST_POISON_ID", + "prev": "STORE_LIST_POISON_PREV", + "hash": "STORE_LIST_POISON_HASH", + "payload": "STORE_LIST_POISON_PAYLOAD", + } + store.conn.execute( + "UPDATE operation_receipts SET id=?, prev_hash=?, receipt_hash=?, payload=? " + "WHERE id=?", + ( + markers["id"], markers["prev"], markers["hash"], + json.dumps({"secret": markers["payload"]}), receipt["id"], + ), + ) + store.conn.commit() + + exported = store.list_receipts(workspace_id=wid) + encoded = json.dumps(exported) + assert all(marker not in encoded for marker in markers.values()) + assert exported[0]["invalid_payload"] is True + assert exported[0]["id"].startswith("redacted_sha256:") + assert exported[0]["prev_hash"].startswith("redacted_sha256:") + assert exported[0]["hash"].startswith("redacted_sha256:") + assert store.verify_receipts(workspace_id=wid)["valid"] is False + + +def test_operation_status_and_nonfinite_metadata_cannot_leak_into_receipts(): + store = Store(":memory:") + wid = store.get_or_create_workspace("w") + secret = "short-secret" + receipt = store.record_receipt( + secret, + workspace_id=wid, + status=secret, + target_count="not-a-number", + metadata={"k": float("nan"), "result_count": float("inf")}, + ) + + encoded = json.dumps(receipt) + assert secret not in encoded + assert receipt["operation"].startswith("sha256:") + assert receipt["status"].startswith("sha256:") + assert receipt["target_count"] == 0 + assert receipt["metadata"] == {} + assert store.verify_receipts(workspace_id=wid)["valid"] is True + + +def test_fixed_terminal_statuses_remain_human_readable(): + store = Store(":memory:") + wid = store.get_or_create_workspace("w") + + failed = store.record_receipt("graph_index", workspace_id=wid, status="failed") + cancelled = store.record_receipt("graph_index", workspace_id=wid, status="cancelled") + + assert failed["status"] == "failed" + assert cancelled["status"] == "cancelled" + + def test_concurrent_receipts_form_one_valid_chain(): store = Store(":memory:") wid = store.get_or_create_workspace("team") @@ -99,6 +187,204 @@ def test_receipt_append_after_truncation_preserves_integrity_failure(): } +def test_append_after_missing_anchor_preserves_public_integrity_marker(): + service = MemoryService.create(":memory:") + first = service.remember("First fact.", workspace="team", scope="workspace") + wid = service._lookup_workspace("team") + service.store.conn.execute( + "DELETE FROM receipt_chain_heads WHERE workspace_id=?", (wid,) + ) + service.store.conn.commit() + + appended = service.store.record_receipt("recall", workspace_id=wid) + exported = service.export_workspace(workspace="team") + + assert appended["prev_hash"] == first["receipt"]["hash"] + assert exported["receipt_chain"]["integrity_error"] == "pre_append_anchor_missing" + assert exported["receipt_verification"]["valid"] is False + + +def test_receipt_append_after_payload_corruption_is_non_bricking_and_stays_invalid(): + store = Store(":memory:") + wid = store.get_or_create_workspace("team") + first = store.record_receipt("remember", workspace_id=wid) + store.conn.execute( + "UPDATE operation_receipts SET payload=payload || ' ' WHERE id=?", + (first["id"],), + ) + store.conn.commit() + + appended = store.record_receipt("recall", workspace_id=wid) + + assert appended["prev_hash"] == first["hash"] + verification = store.verify_receipts(workspace_id=wid) + assert verification["valid"] is False + assert { + "hash_mismatch", "anchor_integrity_error", + } <= {error["error"] for error in verification["errors"]} + + +def test_receipt_fork_has_no_safe_append_head(): + store = Store(":memory:") + wid = store.get_or_create_workspace("team") + first = store.record_receipt("remember", workspace_id=wid) + second = store.record_receipt("recall", workspace_id=wid) + fork = dict(second) + fork.pop("hash") + fork["id"] = new_id("receipt") + fork["prev_hash"] = first["hash"] + fork["ts_ms"] += 1 + payload = json.dumps( + fork, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + receipt_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + store.conn.execute( + "INSERT INTO operation_receipts(id, ts, operation, workspace_id, repo_id, " + "sequence, scope_digest, actor, target_count, status, payload, prev_hash, " + "receipt_hash) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + fork["id"], fork["ts_ms"] / 1000.0, fork["operation"], wid, "", 999, + fork["scope_digest"], fork["actor_digest"], fork["target_count"], + fork["status"], payload, fork["prev_hash"], receipt_hash, + ), + ) + store.conn.commit() + + with pytest.raises(sqlite3.IntegrityError, match="no unique structural head"): + store.record_receipt("link", workspace_id=wid) + + verification = store.verify_receipts(workspace_id=wid) + assert verification["valid"] is False + assert "chain_fork" in {error["error"] for error in verification["errors"]} + + +def test_healthy_receipt_append_does_not_reconstruct_chain(monkeypatch): + store = Store(":memory:") + wid = store.get_or_create_workspace("team") + first = store.record_receipt("remember", workspace_id=wid) + + def unexpected_reconstruction(_workspace_id): + raise AssertionError("healthy append must use the anchored sequence head") + + monkeypatch.setattr(store, "_receipt_chain_state", unexpected_reconstruction) + second = store.record_receipt("recall", workspace_id=wid) + + assert second["prev_hash"] == first["hash"] + assert [ + row["sequence"] for row in store.conn.execute( + "SELECT sequence FROM operation_receipts " + "WHERE workspace_id=? ORDER BY sequence", + (wid,), + ).fetchall() + ] == [1, 2] + + +def test_bounded_receipt_logs_do_not_reconstruct_chain(monkeypatch): + service = MemoryService.create(":memory:") + wid = service.store.get_or_create_workspace("team") + first = service.store.record_receipt("remember", workspace_id=wid) + second = service.store.record_receipt("recall", workspace_id=wid) + + def unexpected_reconstruction(_workspace_id): + raise AssertionError("bounded receipt inspection must use the sequence index") + + monkeypatch.setattr( + service.store, "_receipt_chain_state", unexpected_reconstruction + ) + assert service.store.list_receipts(workspace_id=wid, limit=1) == [ + {**second} + ] + assert service.receipt_log(workspace="team", limit=1)["entries"] == [ + {**second} + ] + assert first["id"] != second["id"] + + +def test_receipt_sequence_is_immutable(): + store = Store(":memory:") + wid = store.get_or_create_workspace("team") + receipt = store.record_receipt("remember", workspace_id=wid) + + with pytest.raises(sqlite3.IntegrityError, match="sequence is immutable"): + store.conn.execute( + "UPDATE operation_receipts SET sequence=2 WHERE id=?", + (receipt["id"],), + ) + store.conn.rollback() + + assert store.verify_receipts(workspace_id=wid)["valid"] is True + + +def test_receipt_chain_survives_physical_row_reordering_and_vacuum(tmp_path): + db = str(tmp_path / "vacuum-receipts.db") + store = Store(db) + wid = store.get_or_create_workspace("team") + receipts = [ + store.record_receipt(operation, workspace_id=wid) + for operation in ("remember", "recall", "link") + ] + for index, receipt in enumerate(receipts): + store.conn.execute( + "UPDATE operation_receipts SET rowid=? WHERE id=?", + (10_000 - index, receipt["id"]), + ) + store.conn.commit() + store.conn.execute("VACUUM") + physical = [ + row["id"] for row in store.conn.execute( + "SELECT id FROM operation_receipts ORDER BY rowid" + ).fetchall() + ] + assert physical == [row["id"] for row in reversed(receipts)] + + assert store.verify_receipts(workspace_id=wid)["valid"] is True + assert [ + row["id"] for row in reversed(store.list_receipts( + workspace_id=wid, limit=10 + )) + ] == [row["id"] for row in receipts] + appended = store.record_receipt("sync", workspace_id=wid) + assert appended["prev_hash"] == receipts[-1]["hash"] + assert store.verify_receipts(workspace_id=wid)["valid"] is True + store.close() + + service = MemoryService.create(db) + try: + exported = service.export_receipts(workspace="team") + assert exported["verification"]["valid"] is True + assert [row["id"] for row in exported["entries"]] == [ + *[row["id"] for row in receipts], + appended["id"], + ] + finally: + service.store.close() + + +def test_reopening_anchored_receipts_does_not_reconstruct_chains(tmp_path, monkeypatch): + db = str(tmp_path / "reopen-receipts.db") + store = Store(db) + wid = store.get_or_create_workspace("team") + store.record_receipt("remember", workspace_id=wid) + store.record_receipt("recall", workspace_id=wid) + store.close() + + calls = [] + original = Store._receipt_chain_state + + def tracked(self, workspace_id): + calls.append(workspace_id) + return original(self, workspace_id) + + monkeypatch.setattr(Store, "_receipt_chain_state", tracked) + reopened = Store(db) + try: + assert calls == [] + assert reopened.verify_receipts(workspace_id=wid)["valid"] is True + assert calls == [wid] + finally: + reopened.close() + + def test_receipt_anchor_migration_normalizes_legacy_null_scope(tmp_path): db = str(tmp_path / "receipts.db") store = Store(db) @@ -109,6 +395,10 @@ def test_receipt_anchor_migration_normalizes_legacy_null_scope(tmp_path): "UPDATE operation_receipts SET workspace_id=NULL, repo_id=NULL WHERE id=?", (receipt["id"],), ) + store.conn.execute("DELETE FROM schema_migrations") + store.conn.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (4, 0)" + ) store.conn.commit() store.close() @@ -124,6 +414,29 @@ def test_receipt_anchor_migration_normalizes_legacy_null_scope(tmp_path): reopened.close() +def test_current_schema_reopen_does_not_recreate_missing_receipt_anchor(tmp_path): + db = str(tmp_path / "missing-anchor.db") + store = Store(db) + wid = store.get_or_create_workspace("team") + store.record_receipt("remember", workspace_id=wid) + store.conn.execute( + "DELETE FROM receipt_chain_heads WHERE workspace_id=?", (wid,) + ) + store.conn.commit() + store.close() + + reopened = Store(db) + try: + verification = reopened.verify_receipts(workspace_id=wid) + assert verification["valid"] is False + assert verification["anchored"] is False + assert "missing_anchor" in { + error["error"] for error in verification["errors"] + } + finally: + reopened.close() + + def test_external_receipt_anchor_detects_rewritten_local_anchor(): store = Store(":memory:") wid = store.get_or_create_workspace("team") @@ -194,3 +507,35 @@ def test_service_records_and_exports_operation_receipts(): assert exported["format"] == "engraphis-receipts/1" assert exported["verification"]["valid"] is True assert {entry["operation"] for entry in exported["entries"]} == {"remember", "recall"} + + +def test_store_and_service_share_strict_receipt_projection(): + service = MemoryService.create(":memory:") + wid = service.store.get_or_create_workspace("acme") + receipt = service.store.record_receipt( + "recall", workspace_id=wid, metadata={"retrieval_profile": "code"} + ) + listed = service.store.list_receipts(workspace_id=wid) + logged = service.receipt_log(workspace="acme")["entries"] + assert listed == logged + assert listed[0]["metadata"]["retrieval_profile"] == "code" + + payload = dict(receipt) + payload.pop("hash") + payload["metadata"] = {"scope": "semantic"} + raw = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + poisoned_hash = hashlib.sha256(raw.encode("utf-8")).hexdigest() + service.store.conn.execute( + "UPDATE operation_receipts SET payload=?, receipt_hash=? WHERE id=?", + (raw, poisoned_hash, receipt["id"]), + ) + service.store.conn.execute( + "UPDATE receipt_chain_heads SET head_hash=? WHERE workspace_id=?", + (poisoned_hash, wid), + ) + service.store.conn.commit() + + listed = service.store.list_receipts(workspace_id=wid) + logged = service.receipt_log(workspace="acme")["entries"] + assert listed == logged + assert listed[0]["invalid_payload"] is True diff --git a/tests/test_resolve.py b/tests/test_resolve.py index 0b4bf04e..0feb914a 100644 --- a/tests/test_resolve.py +++ b/tests/test_resolve.py @@ -52,6 +52,122 @@ def test_resolve_invalidate_on_same_subject_new_content(): assert res.target_id == "mem_old_limit" +def test_resolve_claim_key_invalidates_without_lexical_overlap(): + neighbor = MemoryRecord( + id="mem_old_limit", content="The upstream provider permits 100 calls.", + subject_key="provider-rate-limit", claim_kind="limit", + ) + res = resolve( + "The current cap is 500 requests per minute.", [(0.2, neighbor)], + subject_key="provider-rate-limit", claim_kind="limit", + ) + assert res.op == ResolutionOp.INVALIDATE + assert res.target_id == "mem_old_limit" + + +def test_resolve_never_deduplicates_or_invalidates_conflicting_claim_keys(): + neighbor = MemoryRecord( + id="mem_database_status", + content="The status is enabled.", + subject_key="database", + claim_kind="status", + ) + res = resolve( + "The status is enabled.", + [(0.99, neighbor)], + subject_key="billing", + claim_kind="status", + ) + assert res.op == ResolutionOp.ADD + + +def test_resolve_requires_claim_kind_equality_for_keyed_invalidation(): + neighbor = MemoryRecord( + id="mem_deploy_owner", + content="Production deploys use the platform team.", + subject_key="production-deploy", + claim_kind="owner", + ) + res = resolve( + "Production deploys use the release train.", + [(0.99, neighbor)], + subject_key="production-deploy", + claim_kind="process", + ) + assert res.op == ResolutionOp.ADD + + +def test_shared_claim_key_invalidates_even_when_only_a_number_changes(): + neighbor = MemoryRecord( + id="mem_old_timeout", + content="The request timeout is 5 seconds.", + subject_key="api-timeout", + claim_kind="configured_value", + ) + res = resolve( + "The request timeout is 30 seconds.", + [(0.99, neighbor)], + subject_key="api-timeout", + claim_kind="configured_value", + ) + assert res.op == ResolutionOp.INVALIDATE + assert res.target_id == "mem_old_timeout" + + +def test_exact_claim_identity_outranks_a_more_similar_unkeyed_neighbor(): + keyed = MemoryRecord( + id="mem_keyed", + content="The cap is one hundred.", + subject_key="provider-cap", + claim_kind="limit", + ) + unkeyed = MemoryRecord( + id="mem_unkeyed", + content="The current cap is five hundred.", + ) + res = resolve( + "The current cap is five hundred.", + [(0.2, keyed), (0.999, unkeyed)], + subject_key="provider-cap", + claim_kind="limit", + ) + assert res.op == ResolutionOp.INVALIDATE + assert res.target_id == "mem_keyed" + + +def test_keyed_duplicate_ignores_existing_display_title(): + neighbor = MemoryRecord( + id="mem_titled", + title="API policy", + content="The timeout is 30 seconds.", + subject_key="api-timeout", + claim_kind="configured_value", + ) + res = resolve( + "The timeout is 30 seconds.", + [(0.99, neighbor)], + subject_key="api-timeout", + claim_kind="configured_value", + ) + assert res.op == ResolutionOp.NOOP + assert res.target_id == "mem_titled" + + +def test_new_claim_identity_replaces_instead_of_nooping_unkeyed_duplicate(): + neighbor = MemoryRecord( + id="mem_unkeyed_duplicate", + content="The timeout is 30 seconds.", + ) + res = resolve( + "The timeout is 30 seconds.", + [(0.99, neighbor)], + subject_key="api-timeout", + claim_kind="configured_value", + ) + assert res.op == ResolutionOp.INVALIDATE + assert res.target_id == "mem_unkeyed_duplicate" + + def test_resolve_add_when_related_but_distinct_topic(): # Cause vs. fix: related (both about the checkout race condition) but complementary, # not contradictory — both should be kept. @@ -74,14 +190,14 @@ def test_resolve_picks_best_overlap_among_multiple_neighbors(): # ── paraphrase detection via the embedding-cosine second signal ────────────────── -def test_resolve_paraphrase_invalidates_on_high_cosine_low_overlap(): - # Reworded contradiction: token overlap is far below SUBJECT_TOKEN_JACCARD, but the - # embedding similarity the write path already computed says "same fact, other words". +def test_resolve_paraphrase_relates_on_high_cosine_low_overlap(): + # High cosine alone is topical/paraphrase evidence, not a safe reason to hide + # a live fact. Without a claim key or strong joint evidence it stays related. neighbor = _rec("The API rate limit is one hundred requests every sixty seconds.", id="mem_old_phrasing") candidate = "Calls are capped at 500 per minute for each key." res = resolve(candidate, [(0.95, neighbor)]) - assert res.op == ResolutionOp.INVALIDATE + assert res.op == ResolutionOp.RELATE assert res.target_id == "mem_old_phrasing" assert "paraphrase" in res.reason diff --git a/tests/test_retrieval_policy.py b/tests/test_retrieval_policy.py new file mode 100644 index 00000000..07f9a9c7 --- /dev/null +++ b/tests/test_retrieval_policy.py @@ -0,0 +1,142 @@ +"""Focused contracts for deterministic retrieval-profile routing.""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError +from pathlib import Path + +import pytest + +from engraphis.core.engine import MemoryEngine +from engraphis.core.interfaces import MemoryType, Scope +from engraphis.core.retrieval_policy import ( + DeterministicRetrievalPolicy, + ProfileConfig, + profile_config, +) +from eval.harness import _seed_case_graph, load_dataset + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("balanced", (True, True, True, False)), + ("lexical", (False, True, False, False)), + ("graph", (True, True, True, False)), + ("code", (True, True, True, True)), + ], +) +def test_concrete_profiles_have_stable_arm_configurations( + name: str, expected: tuple[bool, bool, bool, bool] +) -> None: + config = profile_config(name) + + assert (config.vector, config.lexical, config.graph, config.code) == expected + with pytest.raises(FrozenInstanceError): + config.code = False # type: ignore[misc] + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("Trace src/api.py -> Handler.handle()", "code"), + ("Why does checkout depend on auth?", "graph"), + ('Find the exact "RATE_LIMIT" identifier.', "lexical"), + ("What did we decide for the launch?", "balanced"), + ("Why does Handler.handle() call the API_KEY module?", "code"), + ], +) +def test_auto_routing_is_deterministic_and_uses_specific_signals_first( + query: str, expected: str +) -> None: + policy = DeterministicRetrievalPolicy() + + assert policy.profile(query) == expected + assert policy.resolve("auto", query).name == expected + + +@pytest.mark.parametrize( + ("requested", "query", "expected"), + [ + ("balanced", "src/api.py -> Handler.handle()", "balanced"), + ("AUTO", "Find exact RATE_LIMIT", "lexical"), + (" lexical ", "Why does checkout depend on auth?", "lexical"), + ("graph", "Find exact RATE_LIMIT", "graph"), + ("code", "What did we decide for the launch?", "code"), + ], +) +def test_explicit_profile_overrides_auto_routing( + requested: str, query: str, expected: str +) -> None: + policy = DeterministicRetrievalPolicy() + + assert policy.resolve(requested, query) == profile_config(expected) + + +@pytest.mark.parametrize("requested", ["rerank", "semantic", "auto-plus"]) +def test_unknown_requested_profile_is_rejected(requested: str) -> None: + with pytest.raises(ValueError, match="retrieval_profile"): + DeterministicRetrievalPolicy().resolve(requested, "ordinary query") + + +def test_empty_requested_profile_defaults_to_balanced() -> None: + assert DeterministicRetrievalPolicy().resolve("", "src/api.py -> Handler.handle()").name == "balanced" + + +@pytest.mark.parametrize("name", ["auto", "", "unknown"]) +def test_profile_config_requires_a_concrete_profile(name: str) -> None: + with pytest.raises(ValueError, match="resolve to one of"): + profile_config(name) + + +def test_profile_config_returns_an_immutable_value_object() -> None: + config = profile_config("code") + + assert isinstance(config, ProfileConfig) + assert config.name == "code" + + +def test_auto_graph_profile_prioritizes_multi_hop_evidence_without_changing_balanced(): + dataset = ( + Path(__file__).resolve().parents[1] / "eval" / "datasets" / "graph_multihop.jsonl" + ) + case = load_dataset(str(dataset))[0] + assert profile_config("balanced").graph_scale == 1.0 + engine = MemoryEngine.create(":memory:") + workspace_id = engine.store.get_or_create_workspace("eval") + repo_id = engine.store.get_or_create_repo(workspace_id, case["id"]) + _seed_case_graph( + engine.store, + workspace_id=workspace_id, + repo_id=repo_id, + case=case, + ) + by_tag = {} + for memory in case["memories"]: + by_tag[memory["tag"]] = engine.remember( + memory["text"], + workspace_id=workspace_id, + repo_id=repo_id, + mtype=MemoryType.EPISODIC, + scope=Scope.REPO, + resolve_conflicts=False, + ) + + result = engine.recall( + case["questions"][0]["q"], + workspace_id=workspace_id, + repo_id=repo_id, + k=5, + retrieval_profile="auto", + diagnostics=True, + ) + + assert result.retrieval_profile == "graph" + assert by_tag["m_bill"] in {chunk["id"] for chunk in result.chunks} + graph_details = [ + item for item in result.retrieval_trace or [] + if item["id"] == by_tag["m_bill"] + ][0] + assert graph_details["profile_adjusted"]["graph"] == pytest.approx( + graph_details["normalized"]["graph"] * 3.0 + 1.5 + ) diff --git a/tests/test_service.py b/tests/test_service.py index 3ea8bd97..d4148ee2 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -28,6 +28,18 @@ def test_remember_then_recall_roundtrip(): assert any("pnpm" in m["content"] for m in r["memories"]) +def test_service_recall_does_not_reinforce_weak_results_by_default(): + s = _svc() + stored = s.remember("The deployment target is AWS ECS.", workspace="acme", repo="web") + before = s.store.get_memory(stored["id"]).access_count + + s.recall("unrelated lunch menu", workspace="acme", repo="web", k=1) + assert s.store.get_memory(stored["id"]).access_count == before + + s.recall("deployment target", workspace="acme", repo="web", k=1, reinforce=True) + assert s.store.get_memory(stored["id"]).access_count > before + + def test_scope_isolation_by_workspace(): s = _svc() s.remember("Secret alpha fact about widgets.", workspace="alpha") @@ -398,6 +410,75 @@ def test_timeline_orders_chronologically(): assert out["history"][0]["valid_from"] <= out["history"][1]["valid_from"] +def test_service_exposes_world_time_writes_and_point_in_time_recall(): + s = _svc() + old = s.remember( + "The API rate limit is 100 requests per minute.", + workspace="acme", + repo="web", + valid_from=1_000.0, + ) + new = s.remember( + "The API rate limit is 500 requests per minute.", + workspace="acme", + repo="web", + valid_from=2_000.0, + ) + + before = s.recall( + "What is the API rate limit?", + workspace="acme", + repo="web", + as_of=1_500.0, + reinforce=False, + ) + after = s.recall( + "What is the API rate limit?", + workspace="acme", + repo="web", + as_of=2_500.0, + reinforce=False, + ) + assert [memory["id"] for memory in before["memories"]] == [old["id"]] + assert [memory["id"] for memory in after["memories"]] == [new["id"]] + + +@pytest.mark.parametrize( + ("method", "kwargs"), + [ + ("remember", {"content": "A fact.", "workspace": "acme", "valid_from": float("nan")}), + ("remember", {"content": "A fact.", "workspace": "acme", "valid_from": True}), + ("recall", {"query": "A fact.", "workspace": "acme", "as_of": float("inf")}), + ( + "grounded_recall", + {"query": "A fact.", "workspace": "acme", "as_of": "not-a-time"}, + ), + ], +) +def test_service_rejects_invalid_temporal_anchors(method, kwargs): + s = _svc() + with pytest.raises(ValidationError, match="finite timestamp"): + getattr(s, method)(**kwargs) + + +def test_service_rejects_backdated_supersession_as_validation_error(): + s = _svc() + original = s.remember( + "The deployment window is Friday afternoon.", + workspace="acme", + valid_from=2_000.0, + ) + + with pytest.raises(ValidationError, match="cannot predate"): + s.remember( + "The deployment window is Thursday afternoon.", + workspace="acme", + valid_from=1_000.0, + ) + + assert s.store.get_memory(original["id"]).valid_to is None + + def test_recall_proactive_includes_last_session(): s = _svc() s.remember("High importance convention.", workspace="acme", repo="web", importance=0.9) @@ -455,6 +536,105 @@ def test_search_code_requires_repo(): s.search_code("add", workspace="acme", repo="") +def test_service_code_search_honors_bitemporal_anchors(): + """The public service must not append present-day code to historic recall.""" + from engraphis.core.interfaces import MemoryRecord, Scope + + s = _svc() + workspace_id = s.store.get_or_create_workspace("acme") + repo_id = s.store.get_or_create_repo(workspace_id, "api") + symbol_id = s.store.upsert_symbol( + repo_id=repo_id, kind="function", name="legacy_route", fqname="legacy_route", + file="legacy.py", span="1-1", + ) + memory_id = s.store.add_memory(MemoryRecord( + id="", content="legacy_route handled historic requests", title="legacy route", + workspace_id=workspace_id, repo_id=repo_id, scope=Scope.REPO, + valid_from=10.0, ingested_at=10.0, + )) + s.store.link_memory_symbol(repo_id=repo_id, symbol_id=symbol_id, memory_id=memory_id) + for table in ("symbols", "code_memory_links"): + s.store.conn.execute( + f"UPDATE {table} SET valid_from=10, ingested_at=10 WHERE repo_id=?", (repo_id,) + ) + s.store.conn.commit() + s.store.close_validity(memory_id, at=20.0) + s.store.clear_symbols_for_file(repo_id, "legacy.py") + closed_at = s.store.conn.execute( + "SELECT valid_to FROM symbols WHERE id=?", (symbol_id,) + ).fetchone()["valid_to"] + + current = s.search_code("legacy_route", workspace="acme", repo="api") + historic = s.search_code( + "legacy_route", workspace="acme", repo="api", valid_at=15.0, + known_at=float(closed_at) + 1.0, + ) + + assert current["symbols"] == [] + assert [symbol["id"] for symbol in historic["symbols"]] == [symbol_id] + with pytest.raises(ValidationError, match="as_of and valid_at"): + s.search_code( + "legacy_route", workspace="acme", repo="api", as_of=14.0, valid_at=15.0 + ) + + +def test_service_code_export_honors_bitemporal_anchors(): + """Every export companion must be rendered from one anchored graph payload.""" + from engraphis.core.interfaces import MemoryRecord, Scope + + s = _svc() + workspace_id = s.store.get_or_create_workspace("acme") + repo_id = s.store.get_or_create_repo(workspace_id, "api") + symbol_id = s.store.upsert_symbol( + repo_id=repo_id, kind="function", name="legacy_route", fqname="legacy_route", + file="legacy.py", span="1-1", + ) + memory_id = s.store.add_memory(MemoryRecord( + id="", content="legacy_route handled historic requests", title="legacy route", + workspace_id=workspace_id, repo_id=repo_id, scope=Scope.REPO, + valid_from=10.0, ingested_at=10.0, + )) + s.store.link_memory_symbol( + repo_id=repo_id, symbol_id=symbol_id, memory_id=memory_id + ) + for table in ("symbols", "code_memory_links"): + s.store.conn.execute( + f"UPDATE {table} SET valid_from=10, ingested_at=10 WHERE repo_id=?", + (repo_id,), + ) + s.store.conn.commit() + s.store.close_validity(memory_id, at=20.0) + s.store.clear_symbols_for_file(repo_id, "legacy.py") + learned_close = s.store.conn.execute( + "SELECT valid_to FROM symbols WHERE id=?", (symbol_id,) + ).fetchone()["valid_to"] + + current = s.export_code_graph(workspace="acme", repo="api") + before_ingestion = s.export_code_graph( + workspace="acme", repo="api", valid_at=15.0, known_at=9.0, + ) + historical = s.export_code_graph( + workspace="acme", repo="api", as_of=15.0, valid_at=15.0, + known_at=float(learned_close) + 1.0, + ) + + assert current["graph"]["nodes"] == [] + assert before_ingestion["graph"]["nodes"] == [] + assert {row["id"] for row in historical["graph"]["nodes"]} == {symbol_id} + assert {row["memory_id"] for row in historical["graph"]["memory_links"]} == { + memory_id + } + assert "- Symbols: 1" in historical["report_markdown"] + assert "legacy_route" in historical["graph_html"] + assert historical["valid_at"] == 15.0 + assert historical["known_at"] == float(learned_close) + 1.0 + assert historical["historical"] is True + with pytest.raises(ValidationError, match="as_of and valid_at"): + s.export_code_graph( + workspace="acme", repo="api", as_of=14.0, valid_at=15.0 + ) + + # ── folder / file import (dashboard "Import files & folders" section, SECURITY.md §5) ─ def test_import_folder_success(tmp_path, monkeypatch): diff --git a/tests/test_service_graph.py b/tests/test_service_graph.py index 9522fadd..d54fe18e 100644 --- a/tests/test_service_graph.py +++ b/tests/test_service_graph.py @@ -506,6 +506,10 @@ def test_graph_memory_link_fallback_honors_the_requested_as_of_anchor(): "UPDATE memories SET valid_from=?, valid_to=? WHERE id IN (?, ?)", (100.0, 200.0, first["id"], second["id"]), ) + svc.store.conn.execute( + "UPDATE mem_links SET valid_from=100 WHERE a=? AND b=?", + (first["id"], second["id"]), + ) svc.store.conn.commit() assert svc.graph(workspace="acme", backfill=False)["edges"] == [] @@ -516,6 +520,35 @@ def test_graph_memory_link_fallback_honors_the_requested_as_of_anchor(): }] +def test_graph_memory_link_fallback_honors_both_temporal_anchors(): + svc = MemoryService.create(":memory:", graph_extractor="none") + svc.engine.auto_evolve = False + first = svc.remember("Historical alpha", workspace="acme", scope="workspace") + second = svc.remember("Historical beta", workspace="acme", scope="workspace") + svc.link(first["id"], second["id"], workspace="acme", relation="causes") + svc.store.conn.execute( + "UPDATE memories SET valid_from=100, ingested_at=100 WHERE id IN (?, ?)", + (first["id"], second["id"]), + ) + svc.store.conn.execute( + "UPDATE mem_links SET valid_from=100, ingested_at=200 " + "WHERE a=? AND b=?", (first["id"], second["id"]), + ) + svc.store.conn.commit() + + unknown = svc.graph( + workspace="acme", valid_at=150.0, known_at=199.0, backfill=False, + ) + known = svc.graph( + workspace="acme", valid_at=150.0, known_at=200.0, backfill=False, + ) + assert unknown["edges"] == [] + assert known["edges"] == [{ + "from": first["id"], "to": second["id"], + "label": "causes", "layer": "causal", + }] + + def test_graph_lazy_backfill_is_idempotent(): """Re-opening the Graph tab must not duplicate entities.""" svc = MemoryService.create(":memory:", graph_extractor="regex") @@ -684,6 +717,59 @@ def test_graph_as_of_uses_supporting_fact_time_not_entity_backfill_time(): } +def test_graph_applies_independent_world_and_system_time_anchors(): + """Future-ingested public evidence must not leak into a world-time graph.""" + svc = MemoryService.create(":memory:", graph_extractor="none") + wid, ids = _seed_entities( + svc, "acme", + [("Historical Alice", "person"), ("Historical Acme", "organization")], + [], + ) + memory_id = svc.store.add_memory(MemoryRecord( + id="", content="Historical Alice works at Historical Acme.", + workspace_id=wid, scope=Scope.WORKSPACE, + valid_from=100.0, ingested_at=200.0, + )) + edge_id = svc.store.upsert_edge(Edge( + id="", src=ids["Historical Alice"], dst=ids["Historical Acme"], + relation="works_at", workspace_id=wid, + valid_from=100.0, ingested_at=200.0, + provenance={"memory_id": memory_id}, + )) + svc.store.conn.execute( + "UPDATE edge_supports SET valid_from=100, ingested_at=200 " + "WHERE edge_id=? AND memory_id=?", + (edge_id, memory_id), + ) + svc.store.conn.commit() + + unknown = svc.graph( + workspace="acme", valid_at=150.0, known_at=199.0, backfill=False, + ) + known = svc.graph( + workspace="acme", as_of=150.0, valid_at=150.0, + known_at=200.0, backfill=False, + ) + + assert unknown["nodes"] == [] and unknown["edges"] == [] + assert [(edge["from"], edge["to"]) for edge in known["edges"]] == [( + ids["Historical Alice"], ids["Historical Acme"], + )] + assert known["meta"] == { + "nodes_available": 2, + "nodes_complete": True, + "mode": "overview", + "as_of": 150.0, + "valid_at": 150.0, + "known_at": 200.0, + "historical": True, + } + with pytest.raises(ValidationError, match="as_of and valid_at"): + svc.graph( + workspace="acme", as_of=149.0, valid_at=150.0, backfill=False + ) + + def test_forgetting_one_support_keeps_a_multi_source_edge_live(): svc = MemoryService.create(":memory:") wid, ids = _seed_entities( diff --git a/tests/test_store_v4_migration.py b/tests/test_store_v4_migration.py index 6c3a117a..c7decfa1 100644 --- a/tests/test_store_v4_migration.py +++ b/tests/test_store_v4_migration.py @@ -2,12 +2,14 @@ import hashlib import os +import shutil import sqlite3 from pathlib import Path import pytest from engraphis.core.store import Store +from engraphis.core.interfaces import Edge, MemoryRecord, Scope, SearchFilter def _adversarial_link(target: Path, link: Path) -> None: @@ -55,7 +57,7 @@ def test_v3_upgrade_creates_verified_pre_mutation_backup_and_is_idempotent(tmp_p _prepare_v3(db) migrated = Store(str(db)) - assert migrated.schema_version == 4 + assert migrated.schema_version == 5 assert migrated.conn.execute( "SELECT COUNT(*) FROM edge_supports WHERE edge_id='edge_v3'" ).fetchone()[0] == 1 @@ -75,6 +77,203 @@ def test_v3_upgrade_creates_verified_pre_mutation_backup_and_is_idempotent(tmp_p assert hashlib.sha256(backup.read_bytes()).hexdigest() == backup_digest +def test_v4_upgrade_rebuilds_code_history_and_backfills_claim_identity(tmp_path): + """Exercise the physical v4 link-table shape, not just its version marker.""" + db = tmp_path / "v4-code-history.db" + store = Store(str(db)) + workspace_id = store.get_or_create_workspace("acme") + repo_id = store.get_or_create_repo(workspace_id, "api") + memory_id = store.add_memory(MemoryRecord( + id="", content="Production deploys require an approval.", + workspace_id=workspace_id, repo_id=repo_id, scope=Scope.REPO, + metadata={"subject_key": "production-deploy", "claim_kind": "policy"}, + )) + symbol_id = store.upsert_symbol( + repo_id=repo_id, kind="function", name="deploy", fqname="deploy", + file="deploy.py", span="1-1", + ) + # Recreate v4's non-temporal code-link table, including its table-level UNIQUE + # constraint. A migration that only bumps the version cannot pass this test. + for index in ( + "idx_code_mem_live_unique", "idx_code_mem_live_symbol", + "idx_code_mem_symbol", "idx_code_mem_memory", + ): + store.conn.execute(f"DROP INDEX IF EXISTS {index}") + store.conn.execute("DROP TABLE code_memory_links") + store.conn.execute( + "CREATE TABLE code_memory_links (" + "id TEXT PRIMARY KEY, repo_id TEXT NOT NULL, symbol_id TEXT NOT NULL, " + "memory_id TEXT NOT NULL, relation TEXT DEFAULT 'mentions', " + "confidence REAL DEFAULT 1.0, created_at REAL, " + "UNIQUE(repo_id, symbol_id, memory_id, relation))" + ) + store.conn.execute( + "INSERT INTO code_memory_links " + "(id, repo_id, symbol_id, memory_id, relation, confidence, created_at) " + "VALUES ('old_link', ?, ?, ?, 'mentions', 0.7, 10)", + (repo_id, symbol_id, memory_id), + ) + store.conn.execute( + "UPDATE memories SET subject_key='', claim_kind='' WHERE id=?", (memory_id,) + ) + store.conn.execute("DELETE FROM schema_migrations") + store.conn.execute("INSERT INTO schema_migrations(version, applied_at) VALUES (4, 0)") + store.conn.commit() + store.close() + + # A real v4 database may retain its immutable v3→v4 recovery snapshot. + # The v5 migration must not try to overwrite or validate that older file + # against the newer source. + legacy_backup = Path(f"{db}.pre-migration-v4.bak") + shutil.copyfile(db, legacy_backup) + legacy_conn = sqlite3.connect(legacy_backup) + try: + legacy_conn.execute("DELETE FROM schema_migrations") + legacy_conn.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (3, 0)" + ) + legacy_conn.commit() + finally: + legacy_conn.close() + legacy_digest = hashlib.sha256(legacy_backup.read_bytes()).hexdigest() + + upgraded = Store(str(db)) + try: + columns = {row["name"] for row in upgraded.conn.execute( + "PRAGMA table_info(code_memory_links)" + ).fetchall()} + link = upgraded.conn.execute( + "SELECT valid_from, ingested_at FROM code_memory_links WHERE id='old_link'" + ).fetchone() + record = upgraded.get_memory(memory_id) + + assert upgraded.schema_version == 5 + assert Path(f"{db}.pre-migration-v5.bak").is_file() + assert hashlib.sha256(legacy_backup.read_bytes()).hexdigest() == legacy_digest + assert {"valid_from", "valid_to", "ingested_at", "expired_at"} <= columns + assert link["valid_from"] == 10 + assert link["ingested_at"] == 10 + assert record.subject_key == "production-deploy" + assert record.claim_kind == "policy" + + # Retire and recreate the same tuple: the v5 partial uniqueness constraint + # permits history plus one live row, unlike v4's table-level UNIQUE. + upgraded.clear_code_memory_links(repo_id) + recreated = upgraded.link_memory_symbol( + repo_id=repo_id, symbol_id=symbol_id, memory_id=memory_id, + ) + assert recreated != "old_link" + assert upgraded.conn.execute( + "SELECT COUNT(*) AS n FROM code_memory_links WHERE repo_id=? " + "AND symbol_id=? AND memory_id=? AND relation='mentions'", + (repo_id, symbol_id, memory_id), + ).fetchone()["n"] == 2 + finally: + upgraded.close() + + +def test_v4_upgrade_backfills_closed_graph_support_for_historical_recall(tmp_path): + db = tmp_path / "v4-closed-incidence.db" + store = Store(str(db)) + workspace_id = store.get_or_create_workspace("acme") + repo_id = store.get_or_create_repo(workspace_id, "api") + memory_id = store.add_memory(MemoryRecord( + id="", content="Alpha depended on Beta.", + workspace_id=workspace_id, repo_id=repo_id, scope=Scope.REPO, + valid_from=10.0, ingested_at=10.0, + )) + edge_id = store.upsert_edge(Edge( + id="", src="ent_alpha", dst="ent_beta", relation="depends_on", + workspace_id=workspace_id, repo_id=repo_id, + valid_from=10.0, ingested_at=10.0, + provenance={"memory_id": memory_id}, + )) + store.close_validity(memory_id, at=20.0) + store.conn.execute("DELETE FROM memory_entities") + store.conn.execute("DELETE FROM schema_migrations") + store.conn.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (4, 0)" + ) + store.conn.commit() + store.close() + + upgraded = Store(str(db)) + try: + historical = SearchFilter( + workspace_id=workspace_id, + repo_id=repo_id, + valid_at=15.0, + known_at=25.0, + ) + incidence = upgraded.list_memory_entities(historical) + assert { + (row["memory_id"], row["entity_id"]) for row in incidence + } == { + (memory_id, "ent_alpha"), + (memory_id, "ent_beta"), + } + assert upgraded.edge_supports_in_scope( + [edge_id], flt=historical + ) + finally: + upgraded.close() + + +def test_existing_v5_database_with_legacy_memory_links_is_upgraded_safely(tmp_path): + """Repair the short-lived v5 shape without treating old links as ancient facts.""" + db = tmp_path / "v5-direct-link-history.db" + store = Store(str(db)) + store.conn.execute("DROP INDEX IF EXISTS idx_mem_links_temporal") + store.conn.execute("DROP INDEX IF EXISTS idx_mem_links_b") + store.conn.execute("DROP INDEX IF EXISTS idx_mem_links_ab") + store.conn.execute("DROP TABLE mem_links") + store.conn.execute( + "CREATE TABLE mem_links (" + "a TEXT, b TEXT, relation TEXT, layer TEXT DEFAULT 'semantic', " + "reason TEXT DEFAULT '', created_at REAL)" + ) + store.conn.execute( + "INSERT INTO mem_links(a, b, relation, layer, reason, created_at) " + "VALUES ('mem_a', 'mem_b', 'related', 'semantic', 'legacy', 123)" + ) + store.conn.commit() + store.close() + + upgraded = Store(str(db)) + try: + columns = {row["name"] for row in upgraded.conn.execute( + "PRAGMA table_info(mem_links)" + ).fetchall()} + row = upgraded.conn.execute( + "SELECT valid_from, ingested_at, valid_to, expired_at " + "FROM mem_links WHERE a='mem_a'" + ).fetchone() + assert upgraded.schema_version == 5 + assert Path(f"{db}.pre-migration-v5.bak").is_file() + assert {"valid_from", "valid_to", "valid_to_recorded_at", "ingested_at", "expired_at"} <= columns + assert row["valid_from"] == row["ingested_at"] == 123 + assert row["valid_to"] is None and row["expired_at"] is None + finally: + upgraded.close() + + +def test_reopening_v5_does_not_repeat_full_history_migrations(tmp_path, monkeypatch): + db = tmp_path / "already-v5.db" + Store(str(db)).close() + + def unexpected(*_args, **_kwargs): + raise AssertionError("v5 migration transform repeated on an already-v5 database") + + monkeypatch.setattr(Store, "_migrate_code_history_v5", unexpected) + monkeypatch.setattr(Store, "_backfill_claim_identity_v5", unexpected) + monkeypatch.setattr(Store, "_backfill_memory_entities_v5", unexpected) + reopened = Store(str(db)) + try: + assert reopened.schema_version == 5 + finally: + reopened.close() + + def test_migration_transform_failure_rolls_back_and_restart_completes( monkeypatch, tmp_path): db = tmp_path / "restart.db" @@ -102,7 +301,7 @@ def fail_after_prior_schema_work(self): monkeypatch.setattr(Store, "_backfill_edge_supports", original) restarted = Store(str(db)) - assert restarted.schema_version == 4 + assert restarted.schema_version == 5 assert restarted.conn.execute( "SELECT COUNT(*) FROM edge_supports WHERE edge_id='edge_v3'" ).fetchone()[0] == 1 @@ -233,4 +432,4 @@ def require_flush_before_schema(self, previous_version): monkeypatch.setattr(Store, "_apply_schema", require_flush_before_schema) Store(str(db)).close() - assert _version(db) == 4 + assert _version(db) == 5 diff --git a/tests/test_sync.py b/tests/test_sync.py index 34f0ac3c..84f66771 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -90,6 +90,71 @@ def test_serialization_roundtrip_preserves_signature(): assert _signature(r2) == _signature(rec) +def test_sync_roundtrip_preserves_claim_identity_and_closure_knowledge_time(): + rec = MemoryRecord( + id="mem_claim", + content="The cap is 30.", + subject_key="api-cap", + claim_kind="configured_value", + valid_to=200.0, + valid_to_recorded_at=300.0, + ) + restored = dict_to_record(record_to_dict(rec)) + assert restored is not None + assert restored.subject_key == "api-cap" + assert restored.claim_kind == "configured_value" + assert restored.valid_to == 200.0 + assert restored.valid_to_recorded_at == 300.0 + assert _signature(restored) == _signature(rec) + + +def test_sync_merge_keeps_closure_transaction_time_paired_with_earliest_close(): + later_world = MemoryRecord( + id="mem_1", content="x", valid_to=500.0, valid_to_recorded_at=100.0 + ) + earlier_world = MemoryRecord( + id="mem_1", content="x", valid_to=300.0, valid_to_recorded_at=400.0 + ) + merged = merge_record(later_world, earlier_world) + assert merged.valid_to == 300.0 + assert merged.valid_to_recorded_at == 400.0 + assert _signature(merged) == _signature( + merge_record(earlier_world, later_world) + ) + + +def test_sync_v1_omitted_claim_fields_do_not_erase_local_identity(): + store = Store(":memory:") + wid = store.get_or_create_workspace("w") + store.add_memory(MemoryRecord( + id="mem_claim", + content="local", + workspace_id=wid, + subject_key="api-cap", + claim_kind="configured_value", + last_access=1.0, + ingested_at=1.0, + valid_from=1.0, + )) + bundle = { + "format": SYNC_FORMAT, + "version": 1, + "workspace_name": "w", + "repos": {}, + "memories": [{ + "id": "mem_claim", + "content": "remote", + "last_access": 2.0, + "ingested_at": 2.0, + "valid_from": 1.0, + }], + } + SyncEngine(store).apply_bundle(bundle) + restored = store.get_memory("mem_claim") + assert restored.subject_key == "api-cap" + assert restored.claim_kind == "configured_value" + + # ── untrusted-bundle boundary (memory-poisoning threat, SECURITY.md) ────────── def test_apply_rejects_bad_header(): @@ -102,6 +167,33 @@ def test_apply_rejects_bad_header(): se.apply_bundle("i am not a dict") +def test_sync_exports_v2_but_accepts_legacy_v1_without_silent_downgrade(): + engine = MemoryEngine.create(":memory:") + wid = engine.store.get_or_create_workspace("w") + engine.remember( + "The cap is 30.", + workspace_id=wid, + subject_key="api-cap", + claim_kind="configured_value", + resolve_conflicts=False, + ) + syncer = SyncEngine(engine.store) + exported = syncer.export_bundle(wid) + assert exported["version"] == 2 + assert exported["memories"][0]["subject_key"] == "api-cap" + + legacy = dict(exported) + legacy["version"] = 1 + legacy["memories"] = [{ + key: value + for key, value in exported["memories"][0].items() + if key not in {"subject_key", "claim_kind", "valid_to_recorded_at"} + }] + target = Store(":memory:") + report = SyncEngine(target).apply_bundle(legacy) + assert report["added"] == 1 + + def test_apply_clamps_and_drops_bad_rows(): store = Store(":memory:") se = SyncEngine(store) @@ -136,6 +228,50 @@ def test_apply_is_idempotent_on_replay(): assert second["unchanged"] == 2 and second["links_added"] == 0 +def test_sync_reactivates_closed_link_once_and_preserves_history(monkeypatch): + store = Store(":memory:") + syncer = SyncEngine(store) + memories = [ + {"id": "mem_a", "content": "one"}, + {"id": "mem_b", "content": "two"}, + ] + syncer.apply_bundle({ + "format": SYNC_FORMAT, "version": 1, "workspace_name": "w", "repos": {}, + "memories": memories, "mem_links": [], + }) + store.add_link( + "mem_a", "mem_b", relation="related", + valid_from=10.0, valid_to=20.0, valid_to_recorded_at=20.0, + ingested_at=10.0, + ) + bundle = { + "format": SYNC_FORMAT, "version": 1, "workspace_name": "w", "repos": {}, + "memories": memories, + "mem_links": [{"a": "mem_a", "b": "mem_b", "relation": "related"}], + } + monkeypatch.setattr("engraphis.core.store.now_ts", lambda: 40.0) + + first = syncer.apply_bundle(bundle) + replay = syncer.apply_bundle(bundle) + + assert first["links_added"] == 1 + assert replay["links_added"] == 0 + rows = store.conn.execute( + "SELECT valid_from, valid_to FROM mem_links ORDER BY valid_from" + ).fetchall() + assert [(row["valid_from"], row["valid_to"]) for row in rows] == [ + (10.0, 20.0), (40.0, None), + ] + assert [row["valid_from"] for row in store.links_among( + ["mem_a", "mem_b"], + flt=SearchFilter(valid_at=15.0, known_at=50.0), + )] == [10.0] + assert [row["valid_from"] for row in store.links_among( + ["mem_a", "mem_b"], + flt=SearchFilter(valid_at=50.0, known_at=50.0), + )] == [40.0] + + def test_dry_run_writes_nothing(): store = Store(":memory:") se = SyncEngine(store) @@ -820,6 +956,10 @@ def test_link_metadata_merge_converges_independent_of_bundle_order(): right_link = right.get_links("mem_a")[0] assert (left_link["layer"], left_link["reason"]) == ("causal", "zeta") assert (right_link["layer"], right_link["reason"]) == ("causal", "zeta") + assert left_sync.apply_bundle(causal)["links_updated"] == 0 + assert right_sync.apply_bundle(semantic)["links_updated"] == 0 + assert left.conn.execute("SELECT COUNT(*) FROM mem_links").fetchone()[0] == 2 + assert right.conn.execute("SELECT COUNT(*) FROM mem_links").fetchone()[0] == 2 def test_deeply_nested_json_does_not_crash_sync_decoding(tmp_path): diff --git a/tests/test_workspace_ops.py b/tests/test_workspace_ops.py index b4b66233..47ddf6b6 100644 --- a/tests/test_workspace_ops.py +++ b/tests/test_workspace_ops.py @@ -9,7 +9,7 @@ import pytest -from engraphis.core.interfaces import Edge, GraphLayer, Node +from engraphis.core.interfaces import Edge, GraphLayer, Node, SearchFilter from engraphis.service import MemoryService, ValidationError @@ -126,6 +126,55 @@ def test_delete_removes_normalized_evidence_without_orphaning_shared_edge(): assert supports == [{"memory_id": retained, "valid_to": None}] +def test_delete_removes_sparse_memory_entity_incidence(): + svc = MemoryService.create(":memory:", graph_extractor="none") + memory_id = svc.remember( + "Delete-only graph evidence.", workspace="a", scope="workspace" + )["id"] + wid = _wsid(svc, "a") + entity_id = svc.store.upsert_entity(Node( + id="", name="Delete Beacon", ntype="concept", workspace_id=wid, + )) + svc.store.link_memory_entity( + memory_id=memory_id, entity_id=entity_id, workspace_id=wid, + repo_id=None, source_kind="explicit", + ) + assert svc.store.conn.execute( + "SELECT COUNT(*) FROM memory_entities WHERE workspace_id=?", (wid,) + ).fetchone()[0] >= 1 + + svc.delete_workspace("a") + + assert svc.store.conn.execute( + "SELECT COUNT(*) FROM memory_entities" + ).fetchone()[0] == 0 + + +def test_delete_removes_receipts_and_chain_anchor_for_only_that_workspace(): + svc = _svc() + svc.remember("Delete this workspace.", workspace="a", scope="workspace") + svc.remember("Retain this workspace.", workspace="b", scope="workspace") + wid_a = _wsid(svc, "a") + wid_b = _wsid(svc, "b") + c = svc.store.conn + assert c.execute( + "SELECT COUNT(*) FROM operation_receipts WHERE workspace_id=?", (wid_a,) + ).fetchone()[0] == 1 + assert c.execute( + "SELECT COUNT(*) FROM receipt_chain_heads WHERE workspace_id=?", (wid_a,) + ).fetchone()[0] == 1 + + svc.delete_workspace("a") + + for table in ("operation_receipts", "receipt_chain_heads"): + assert c.execute( + f"SELECT COUNT(*) FROM {table} WHERE workspace_id=?", (wid_a,) + ).fetchone()[0] == 0 + assert c.execute( + f"SELECT COUNT(*) FROM {table} WHERE workspace_id=?", (wid_b,) + ).fetchone()[0] == 1 + + def test_merge_folds_memories_and_removes_source(): svc = _svc() a1 = svc.remember("Alpha one fact.", workspace="a", scope="workspace")["id"] @@ -142,6 +191,76 @@ def test_merge_folds_memories_and_removes_source(): assert svc.store.get_memory(a1).content == "Alpha one fact." +def test_merge_discards_source_receipt_chain_without_touching_target_chain(): + svc = _svc() + svc.remember("Source fact.", workspace="a", scope="workspace") + svc.remember("Target fact.", workspace="b", scope="workspace") + wid_src = _wsid(svc, "a") + wid_dst = _wsid(svc, "b") + target_before = svc.store.verify_receipts(workspace_id=wid_dst) + + svc.merge_workspaces("a", "b") + + assert svc.store.verify_receipts(workspace_id=wid_dst) == target_before + assert svc.store.conn.execute( + "SELECT COUNT(*) FROM operation_receipts WHERE workspace_id=?", + (wid_src,), + ).fetchone()[0] == 0 + assert svc.store.conn.execute( + "SELECT COUNT(*) FROM receipt_chain_heads WHERE workspace_id=?", + (wid_src,), + ).fetchone()[0] == 0 + assert svc.store.conn.execute( + "SELECT COUNT(*) FROM operation_receipts WHERE workspace_id=?", + (wid_dst,), + ).fetchone()[0] == target_before["count"] + + +def test_merge_rehomes_incidence_and_preserves_graph_recall(): + svc = MemoryService.create(":memory:", graph_extractor="none") + memory_id = svc.remember( + "Opaque graph payload.", workspace="a", scope="workspace" + )["id"] + svc.create_workspace("b") + wid_src = _wsid(svc, "a") + wid_dst = _wsid(svc, "b") + source_entity = svc.store.upsert_entity(Node( + id="", name="Merge Beacon", ntype="concept", workspace_id=wid_src, + )) + target_entity = svc.store.upsert_entity(Node( + id="", name="Merge Beacon", ntype="concept", workspace_id=wid_dst, + )) + svc.store.link_memory_entity( + memory_id=memory_id, entity_id=source_entity, workspace_id=wid_src, + repo_id=None, source_kind="explicit", + ) + + svc.merge_workspaces("a", "b") + + incidence = svc.store.conn.execute( + "SELECT memory_id, entity_id, workspace_id FROM memory_entities " + "WHERE memory_id=? AND source_kind='explicit' " + "AND valid_to IS NULL AND expired_at IS NULL", + (memory_id,), + ).fetchone() + assert dict(incidence) == { + "memory_id": memory_id, + "entity_id": target_entity, + "workspace_id": wid_dst, + } + assert svc.store.conn.execute( + "SELECT COUNT(*) FROM memory_entities WHERE workspace_id=?", (wid_src,) + ).fetchone()[0] == 0 + result = svc.engine.recall_engine.recall( + "How is Merge Beacon related?", + SearchFilter(workspace_id=wid_dst), + k=5, reinforce=False, retrieval_profile="graph", diagnostics=True, + ) + assert memory_id in {chunk["id"] for chunk in result.chunks} + trace = next(item for item in result.retrieval_trace if item["id"] == memory_id) + assert trace["raw"]["graph"] > 0 + + def test_merge_folds_colliding_repos_without_duplicating(): svc = _svc() svc.remember("A web note.", workspace="a", repo="web", scope="repo") @@ -264,11 +383,18 @@ def test_merge_does_not_duplicate_symbols_for_overlapping_files(): svc.merge_workspaces("a", "b") # src is newer → its snapshot wins - rows = c.execute( - "SELECT content_hash FROM symbols WHERE repo_id=? AND file='deploy.py'", + # v5 keeps the displaced target snapshot as closed history. The Store's default + # temporal read is the production invariant: exactly one live symbol, from the + # newer source snapshot. + rows = svc.store.list_symbols(dst_repo) + assert [r["content_hash"] for r in rows if r["file"] == "deploy.py"] == ["src-symbol"] + historical = [dict(row) for row in c.execute( + "SELECT content_hash, valid_to FROM symbols WHERE repo_id=? AND file='deploy.py'", (dst_repo,), - ).fetchall() - assert [r["content_hash"] for r in rows] == ["src-symbol"] + ).fetchall()] + assert {row["content_hash"] for row in historical} == {"src-symbol", "dst-symbol"} + assert any(row["content_hash"] == "dst-symbol" and row["valid_to"] is not None + for row in historical) assert c.execute( "SELECT content_hash FROM code_files WHERE repo_id=? AND file='deploy.py'", (dst_repo,), @@ -300,11 +426,8 @@ def test_merge_does_not_duplicate_symbols_for_overlapping_files(): svc.merge_workspaces("c", "b") - rows = c.execute( - "SELECT content_hash FROM symbols WHERE repo_id=? AND file='deploy.py'", - (dst_repo,), - ).fetchall() - assert [r["content_hash"] for r in rows] == ["src-symbol"] + rows = svc.store.list_symbols(dst_repo) + assert [r["content_hash"] for r in rows if r["file"] == "deploy.py"] == ["src-symbol"] assert c.execute( "SELECT content_hash FROM code_files WHERE repo_id=? AND file='deploy.py'", (dst_repo,), @@ -566,7 +689,8 @@ def test_copy_clones_vectors_fts_links_entities_and_edges(): repo="infra", scope="repo")["id"] svc.link( m1, m2, workspace="a", relation="related", - layer="causal", reason="deployment depends on the database", + layer="causal", + reason=f"deployment {m1} depends on the database record {m2}", ) src_repo_id = svc.store.conn.execute( "SELECT id FROM repos WHERE workspace_id=?", (_wsid(svc, "a"),) @@ -603,6 +727,10 @@ def test_copy_clones_vectors_fts_links_entities_and_edges(): id="", src=deploy, dst=database, relation="depends_on", layer=GraphLayer.CAUSAL, workspace_id=wid_src, repo_id=src_repo_id, )) + svc.store.link_memory_entity( + memory_id=m1, entity_id=database, workspace_id=wid_src, + repo_id=src_repo_id, source_kind="explicit", confidence=0.9, + ) svc.copy_workspace("a", new_name="a2") wid_dst = _wsid(svc, "a2") @@ -644,7 +772,10 @@ def test_copy_clones_vectors_fts_links_entities_and_edges(): (new_a, new_b, new_b, new_a)).fetchone() assert linked is not None assert linked["layer"] == "causal" - assert linked["reason"] == "deployment depends on the database" + assert linked["reason"] == ( + f"deployment {new_a} depends on the database record {new_b}" + ) + assert m1 not in linked["reason"] and m2 not in linked["reason"] copied_entities = { (row["name"], row["etype"]): row["id"] for row in c.execute( @@ -710,6 +841,142 @@ def test_copy_clones_vectors_fts_links_entities_and_edges(): assert len(graph["files"]) == 1 assert len(graph["memory_links"]) == 1 + # Sparse graph incidence is cloned and powers the graph arm without rescanning + # copied memory prose or retaining either source endpoint id. + copied_incidence = c.execute( + "SELECT memory_id, entity_id, workspace_id, repo_id " + "FROM memory_entities WHERE workspace_id=? AND source_kind='explicit'", + (wid_dst,), + ).fetchone() + assert dict(copied_incidence) == { + "memory_id": new_a, + "entity_id": copied_entities[("Postgres", "database")], + "workspace_id": wid_dst, + "repo_id": copied_repo, + } + graph_recall = svc.engine.recall_engine.recall( + "How is Postgres related?", + SearchFilter( + workspace_id=wid_dst, repo_id=copied_repo, include_ancestors=True, + ), + k=5, reinforce=False, retrieval_profile="graph", diagnostics=True, + ) + assert new_a in {chunk["id"] for chunk in graph_recall.chunks} + trace = next( + item for item in graph_recall.retrieval_trace if item["id"] == new_a + ) + assert trace["raw"]["graph"] > 0 + + +def test_copy_preserves_schema_v5_temporal_and_claim_fields(): + svc = MemoryService.create(":memory:", graph_extractor="none") + memory_id = svc.remember( + "Historic claim tied to archived code.", + workspace="a", repo="infra", scope="repo", + subject_key="deploy.target", claim_kind="configured_value", + )["id"] + wid_src = _wsid(svc, "a") + repo_src = svc.store.conn.execute( + "SELECT id FROM repos WHERE workspace_id=?", (wid_src,) + ).fetchone()["id"] + entity_id = svc.store.upsert_entity(Node( + id="", name="Historic Target", ntype="system", + workspace_id=wid_src, repo_id=repo_src, + )) + edge_id = svc.store.upsert_edge(Edge( + id="", src=entity_id, dst=entity_id, relation="documents", + workspace_id=wid_src, repo_id=repo_src, + provenance={"memory_id": memory_id}, + )) + symbol_id = svc.store.upsert_symbol( + repo_id=repo_src, kind="function", name="retired", fqname="retired", + file="retired.py", span="1-1", + ) + code_edge_id = svc.store.add_code_edge( + repo_id=repo_src, src="retired", dst="archive", + relation="calls", file="retired.py", line=1, + ) + code_link_id = svc.store.link_memory_symbol( + repo_id=repo_src, symbol_id=symbol_id, memory_id=memory_id + ) + incidence_id = svc.store.link_memory_entity( + memory_id=memory_id, entity_id=entity_id, workspace_id=wid_src, + repo_id=repo_src, source_kind="explicit", + ) + c = svc.store.conn + for table, identity in ( + ("memories", memory_id), + ("edges", edge_id), + ("symbols", symbol_id), + ("code_edges", code_edge_id), + ("code_memory_links", code_link_id), + ("memory_entities", incidence_id), + ): + c.execute( + f"UPDATE {table} SET valid_from=10, valid_to=20, " + "valid_to_recorded_at=30, ingested_at=5, expired_at=40 WHERE id=?", + (identity,), + ) + c.execute( + "UPDATE edge_supports SET valid_from=10, valid_to=20, " + "valid_to_recorded_at=30, ingested_at=5, expired_at=40 WHERE edge_id=?", + (edge_id,), + ) + c.commit() + + svc.copy_workspace("a", new_name="a2") + wid_dst = _wsid(svc, "a2") + repo_dst = c.execute( + "SELECT id FROM repos WHERE workspace_id=?", (wid_dst,) + ).fetchone()["id"] + expected_temporal = { + "valid_from": 10.0, "valid_to": 20.0, + "valid_to_recorded_at": 30.0, "ingested_at": 5.0, "expired_at": 40.0, + } + copied_memory = dict(c.execute( + "SELECT id, subject_key, claim_kind, valid_from, valid_to, " + "valid_to_recorded_at, ingested_at, expired_at " + "FROM memories WHERE workspace_id=?", + (wid_dst,), + ).fetchone()) + copied_memory_id = copied_memory.pop("id") + assert copied_memory.pop("subject_key") == "deploy.target" + assert copied_memory.pop("claim_kind") == "configured_value" + assert copied_memory == expected_temporal + + checks = ( + ("edges", "workspace_id=? AND relation='documents'", wid_dst), + ("symbols", "repo_id=? AND fqname='retired'", repo_dst), + ("code_edges", "repo_id=? AND file='retired.py'", repo_dst), + ("code_memory_links", "repo_id=? AND memory_id=?", (repo_dst, copied_memory_id)), + ("memory_entities", "workspace_id=? AND source_kind='explicit'", wid_dst), + ) + for table, predicate, values in checks: + params = values if isinstance(values, tuple) else (values,) + row = dict(c.execute( + f"SELECT valid_from, valid_to, valid_to_recorded_at, " + f"ingested_at, expired_at FROM {table} WHERE {predicate}", + params, + ).fetchone()) + assert row == expected_temporal + copied_edge_id = c.execute( + "SELECT id FROM edges WHERE workspace_id=?", (wid_dst,) + ).fetchone()["id"] + assert dict(c.execute( + "SELECT valid_from, valid_to, valid_to_recorded_at, ingested_at, expired_at " + "FROM edge_supports WHERE edge_id=?", + (copied_edge_id,), + ).fetchone()) == expected_temporal + incidence = c.execute( + "SELECT memory_id, entity_id, repo_id FROM memory_entities WHERE workspace_id=?", + (wid_dst,), + ).fetchone() + assert incidence["memory_id"] == copied_memory_id + assert incidence["repo_id"] == repo_dst + assert c.execute( + "SELECT workspace_id FROM entities WHERE id=?", (incidence["entity_id"],) + ).fetchone()["workspace_id"] == wid_dst + def test_copy_rejects_missing_source_and_colliding_new_name(): svc = _svc() From c1272df5de3f38cbf6453487c1935118afc67074 Mon Sep 17 00:00:00 2001 From: Jaixii Date: Thu, 30 Jul 2026 00:29:53 -0400 Subject: [PATCH 02/21] feat(eval): add benchmark v2 and reproducible evidence --- BENCHMARKS.md | 109 ++- MANIFEST.in | 5 +- eval/BASELINES.md | 21 + eval/ablation.py | 73 +- eval/benchmark.py | 1034 ++++++++++++++++++++ eval/configs/longmemeval_v2_engraphis.json | 15 + eval/datasets/adversarial.jsonl | 1 + eval/external.py | 81 +- eval/harness.py | 790 ++++++++++++++- eval/longmemeval_v2.py | 615 ++++++++++++ eval/metrics.py | 91 ++ eval/performance.py | 764 +++++++++++++++ eval/run_longmemeval_v2.py | 110 +++ pyproject.toml | 3 +- scripts/release_evidence.py | 369 +++++++ tests/test_benchmark_adversarial.py | 16 + tests/test_benchmark_evidence.py | 650 ++++++++++++ tests/test_benchmark_longmemeval_v2.py | 413 ++++++++ tests/test_eval_external.py | 24 +- tests/test_eval_harness.py | 393 +++++++- tests/test_eval_performance.py | 211 ++++ tests/test_release_evidence.py | 228 +++++ 22 files changed, 5934 insertions(+), 82 deletions(-) create mode 100644 eval/BASELINES.md create mode 100644 eval/benchmark.py create mode 100644 eval/configs/longmemeval_v2_engraphis.json create mode 100644 eval/datasets/adversarial.jsonl create mode 100644 eval/longmemeval_v2.py create mode 100644 eval/performance.py create mode 100644 eval/run_longmemeval_v2.py create mode 100644 scripts/release_evidence.py create mode 100644 tests/test_benchmark_adversarial.py create mode 100644 tests/test_benchmark_evidence.py create mode 100644 tests/test_benchmark_longmemeval_v2.py create mode 100644 tests/test_eval_performance.py create mode 100644 tests/test_release_evidence.py diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 1da576fd..3d4f58db 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -19,15 +19,25 @@ and stated everywhere the numbers appear (`eval/external.py`). them through the *real* `MemoryEngine` write path (conflict resolution + evolution) and hybrid recall with a real sentence-transformers embedder. It reports `recall_at_k` / `hit_at_k` / `answer_token_recall` — i.e. *did the evidence come back*, not *did an LLM answer correctly*. + It retains source categories and abstention/no-evidence questions as explicit exclusions from + retrieval-only aggregates rather than silently dropping them. `eval.longmemeval_v2` is a local, + text-only adapter for the official LongMemEval-V2 `insert(trajectory)` / `query(query, + query_image=None)` memory interface; it does not download data or call a model. - **Grounded** — `eval/grounded.py`: answerable → cite, off-topic → abstain. - **Chunking (quality per token)** — `eval/chunking_eval.py` over `eval/datasets/longdoc.jsonl` ingests a multi-topic corpus twice — one memory per document (`whole`) vs. sub-file `ChunkingExtractor` (`chunked`) — and queries both through the real recall pipeline. This is the first cut of the context-reduction metric (item 3 below). On the deterministic embedder: - **recall@5 1.000 for both, at ~73% fewer context tokens (826 → 224) and ~4× smaller + **recall@5 1.000 for both, at ~73% fewer context tokens (809 → 219) and ~4× smaller tokens-to-evidence (162 → 42).** Pass `--embed-model sentence-transformers/all-MiniLM-L6-v2` for a real retrieval number (recall should then favour chunked on larger corpora, not just tie). +- **Full-pipeline latency + quality** — `eval/performance.py` times the shipped semantic + + lexical + graph + fusion + scoring + rerank + packing path after warmup, with reinforcement + disabled so repeated measurements do not mutate their corpus. It reports p50/p95/p99 latency, + retrieval quality, and packed context tokens in one JSON-safe schema. `--filler-memories` + provides deterministic corpus scaling, and every report records the runtime, architecture, + embedder, vector backend, corpus size, warmups, and iteration count. ### Reproduce @@ -38,6 +48,11 @@ python -m eval.harness --dataset eval/datasets/sample.jsonl --k 5 python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5 python -m eval.harness --dataset eval/datasets/graph_multihop.jsonl --k 5 python -m eval.ablation +python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 --iterations 10 +python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 \ + --iterations 5 --filler-memories 1000 +# Canonical latency/resource protocol: requires >=1,000 queries and five processes. +python -m eval.performance --dataset fixed-1000-plus.jsonl --acceptance-matrix --processes 5 # Real retrieval numbers (downloads all-MiniLM-L6-v2) python -m eval.external --dataset longmemeval_s.json --format longmemeval --k 10 @@ -47,22 +62,86 @@ python -m eval.external --dataset locomo10.json --format locomo --k 10 ## What we do NOT yet claim - **No end-to-end QA accuracy.** Official LoCoMo / LongMemEval QA scores depend on an answering model and evaluator. Engraphis isolates retrieval and does not present that result as end-to-end answer accuracy. -- **No published latency.** There is no measured p50/p95 recall latency in-repo; we have not - measured our equivalent. The Rust hot path (Phase 6) is not started. +- **No hosted-service latency comparison.** The in-repo p50/p95/p99 benchmark covers the local + reference pipeline and records its environment; unlike environments are not compared. - **No neutral third-party ranking.** We have not run an external eval platform. -## Plan to produce publishable numbers +Every publishable run should emit the `engraphis-benchmark/v2` envelope: dataset/config hashes, +per-question records, explicit exclusions, fixed-budget context curves, and deterministic +stratified or paired bootstrap confidence intervals. Every run names its token counter. +Noncanonical offline fixtures may identify a deterministic estimate; canonical public evidence +requires the exact pinned reader tokenizer and immutable model revision. The lightweight CI +fixtures validate that machinery; they are not a claim about external benchmark performance. + +The benchmark context metric reads strict recall usage fields rather than inferring prompt size: +`budget_tokens`, `context_tokens`, `source_tokens`, `saved_tokens`, `savings_ratio`, +`packed_count`, `omitted_count`, and `token_counter`. Use `engraphis_recall_context` for a +hard-budget prompt packet; legacy `engraphis_recall` remains available in full or compact response +mode for compatibility. + +### Canonical public artifacts + +Use `python -m eval.benchmark --input report.json --output artifacts/run.json` to validate a +report and write sorted, immutable JSON plus `run.json.sha256`. The command permits an identical +retry but refuses to replace a different artifact at the same path. For an official +LongMemEval-V2 run, add `--canonical`: this requires a profile with an exact benchmark repository +revision, dataset revision, reader model revision, and embedding model revision. The checked-in +profile pins immutable upstream commits; replacing any revision with a mutable tag fails +validation. Canonical profiles label the baseline (`no_retrieval`, `lexical_only`, `dense_only`, +`dense_lexical_rrf`, `full_hybrid`, `full_history`, `no_graph`, `no_reranker`, +`no_temporal_resolution`, or `whole_document`) and declare the required fixed context-budget +matrix: 256, 512, 1024, 2048, and 4096 tokens. Canonical in-repo reports rerun every question at +all five budgets and validate each aggregate against its per-question evidence. The checked-in +LongMemEval-V2 memory-module configuration sets the official adapter's operating point to 1,024 +tokens; that single official point must not be presented as a five-point curve. + +`eval.external --canonical` refuses `--limit` and rejects a normalized output that omitted source +cases. Retrieval-only abstention/no-evidence records remain visible in the artifact's +`exclusions`; they are not counted as evidence-retrieval scores. + +### LongMemEval-V2 memory-module adapter + +`eval.longmemeval_v2.EngraphisLongMemEvalV2Memory` follows the official +`memory_modules.memory.Memory` interface at LongMemEval-V2 commit +`6f020ac2fc3275e46c706d3406e02c3ed79b7be2`. When imported in that environment, its +`@register_memory` decorator registers `memory_type="engraphis"`; use the checked-in +[`eval/configs/longmemeval_v2_engraphis.json`](eval/configs/longmemeval_v2_engraphis.json) +with the official harness. The config pins `Qwen/Qwen3-Embedding-8B` to revision +`1d8ad4ca9b3dd8059ad90a75d4983776a23d44af`; mutable embedding revisions are rejected, and a +canonical adapter run fails instead of relabeling the deterministic offline fallback as Qwen. +Run `python -m eval.run_longmemeval_v2` with the official harness arguments and the pinned +checkout on `PYTHONPATH`. This wrapper performs the upstream registry import in the required order +before delegating to `evaluation.harness`; a direct upstream invocation must otherwise import +`eval.longmemeval_v2` before calling `build_memory`. + +The checked-in configuration is canonical only when the adapter resolves the pinned Qwen reader +processor at `c202236235762e1c871ad0ccb60c8ee5ba337b9a`. The wrapper also forces the audited +official harness's otherwise-unpinned `AutoProcessor` call to that same revision. It refuses to +start if the optional processor dependency or immutable revision is unavailable; the local regex +counter is never silently relabeled as a reader budget. The recorded budget counts each returned +context item's content with that reader tokenizer (without prompt framing or inter-item +separators), so it is a hard **evidence-item content** budget, not a claim about total chat-prompt +tokens. Packed sources are returned as separate context items, preserving the largest fitting +evidence prefix instead of dropping one oversized monolithic item. The adapter does not download +benchmark data or call the reader/evaluator; the official harness owns those steps. + +## Next steps for external publishable numbers 1. **Add a QA layer to `eval/external.py`.** Optional answering model + judge on top of the - existing retrieval pipeline, so we can report end-to-end accuracy on the same datasets the - field quotes — reusing the retrieval harness underneath. -2. **Measure recall latency.** Instrument `RecallEngine.recall()` end to end (parallel arms → - RRF → score → rerank → pack) and publish p50/p95 on a fixed corpus and machine class. -3. **Adopt a context-reduction metric.** Report **recall@k against tokens injected** — recall - at a fixed token budget, and tokens-to-first-correct-evidence. It is a natural fit: Engraphis - already does token-budget context packing in recall and already reports a **compaction** - number from consolidation (`core/consolidate.py::_compaction`). Wire those two together into - one "quality per token" curve — arguably our strongest story, since decay + consolidation - are built to raise it. -4. **Run an external eval platform** for a neutral comparison once (1)–(3) exist. + existing retrieval pipeline, so the official datasets can report end-to-end accuracy while + reusing the retrieval harness underneath. +2. **Publish production-backend latency.** Run `eval/performance.py` with the real embedder and + sqlite-vec/backend configuration on a fixed machine class and corpus scale. +3. **Run the fixed-budget curve on the complete official datasets.** The v2 harness now measures + every question at 256, 512, 1,024, 2,048, and 4,096 evidence tokens and validates the + per-question records, aggregates, and pinned reader-tokenizer identity. Publish the curve only + after complete official runs produce immutable artifacts for every point. +4. **Run an external evaluation platform** once (1)–(3) exist. + +## Evaluation question +The predeclared question is whether the full vector + lexical/BM25 + sparse PPR graph + calibrated +rerank pipeline, bi-temporal resolution, and grounded abstention produce higher evidence recall +per injected token than the registered baselines. The answer must come from a complete, +machine-readable artifact with paired confidence intervals; otherwise the release reports +“no demonstrated improvement.” diff --git a/MANIFEST.in b/MANIFEST.in index 8789a2e6..7ceee97b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,10 +8,13 @@ recursive-include engraphis/classic_assets/vendor * recursive-include engraphis/dashboard_assets *.html *.css *.js *.png *.ico recursive-include engraphis/dashboard_assets/vendor * include engraphis/commercial_manifest.json -include LICENSE NOTICE README.md CHANGELOG.md +include LICENSE NOTICE README.md CHANGELOG.md BENCHMARKS.md include pyproject.toml include .env.example requirements.txt include docker-entrypoint.sh Dockerfile docker-compose.yml include railway.json +recursive-include eval *.py +include eval/BASELINES.md +recursive-include eval/configs *.json recursive-include eval/datasets *.jsonl recursive-include tests *.py diff --git a/eval/BASELINES.md b/eval/BASELINES.md new file mode 100644 index 00000000..54e3a2ca --- /dev/null +++ b/eval/BASELINES.md @@ -0,0 +1,21 @@ +# Harness baseline semantics + +`eval.harness` records a `baseline_execution` object in every report. A label is +not a display-only alias: it either changes the executed path or fails before an +artifact is returned when the fixture/runtime cannot represent the claim. + +- `dense_lexical_rrf` uses vector and lexical arms with graph disabled. It is + explicitly recorded as equivalent to `no_graph` because the current pipeline + applies RRF to every multi-arm retrieval configuration. +- `full_history` returns every stored version in chronological source order, + including invalidated records; `whole_document` returns each case's raw + `document`. Neither query-selects or truncates context, so both reject an + insufficient explicit token budget. +- `no_reranker` requires a supplied non-identity reranker and disables it for + the run. `no_temporal_resolution` requires an explicit repeated + `subject_key`/`claim_kind` fixture and writes without conflict resolution. + +Rows with `answerable` labels remain excluded from retrieval metrics when they +have no gold evidence. Pass `--grounded` to score those same labels with +grounded-answer and abstention precision/recall/F1; otherwise the v2 metrics +publish an explicit unavailable reason rather than an implied zero. diff --git a/eval/ablation.py b/eval/ablation.py index 28a04425..dfd4b323 100644 --- a/eval/ablation.py +++ b/eval/ablation.py @@ -10,6 +10,7 @@ from __future__ import annotations from pathlib import Path +import re from engraphis.backends import DeterministicEmbedder, NumpyVectorIndex from engraphis.backends.reranker import IdentityReranker @@ -20,7 +21,9 @@ from eval.harness import load_dataset -def _seed_graph(store: Store, *, workspace_id: str, repo_id: str, case: dict) -> None: +def _seed_graph( + store: Store, *, workspace_id: str, repo_id: str, case: dict, +) -> dict[str, str]: """Persist readable dataset edges with the entity IDs returned by the store.""" entity_ids: dict[str, str] = {} for entity in case.get("entities", []): @@ -42,9 +45,39 @@ def _seed_graph(store: Store, *, workspace_id: str, repo_id: str, case: dict) -> relation=(edge[2] if len(edge) > 2 else "rel"), workspace_id=workspace_id, repo_id=repo_id, )) + return entity_ids + + +def _link_fixture_mentions( + store: Store, + *, + memory_id: str, + text: str, + entity_ids: dict[str, str], + workspace_id: str, + repo_id: str, +) -> None: + """Materialize the same exact-mention incidence used by production writes.""" + for name, entity_id in entity_ids.items(): + if re.search(r"(? float: +def _score( + dataset: list[dict], + *, + k: int, + hybrid: bool, + graph_mode: str = "ppr", + retrieval_profile: str = "balanced", +) -> float: emb = DeterministicEmbedder(256) per = [] for case in dataset: @@ -54,17 +87,32 @@ def _score(dataset: list[dict], *, k: int, hybrid: bool, graph_mode: str = "ppr" index = NumpyVectorIndex(store) # Seed the entity graph when the case provides one (optional keys), so the graph # arm has something to walk — mirrors what production extraction populates. - _seed_graph(store, workspace_id=wid, repo_id=rid, case=case) + entity_ids = _seed_graph(store, workspace_id=wid, repo_id=rid, case=case) engine = RecallEngine(store, emb, index, IdentityReranker(), graph_mode=graph_mode) tag_by_id = {} for m in case["memories"]: mid = store.add_memory(MemoryRecord( id="", content=m["text"], mtype=MemoryType.EPISODIC, scope=Scope.REPO, workspace_id=wid, repo_id=rid, embedding=emb.embed([m["text"]])[0])) + _link_fixture_mentions( + store, + memory_id=mid, + text=m["text"], + entity_ids=entity_ids, + workspace_id=wid, + repo_id=rid, + ) tag_by_id[mid] = m.get("tag") for q in case["questions"]: if hybrid: - ids = [c["id"] for c in engine.recall(q["q"], SearchFilter(workspace_id=wid), k=k).chunks] + ids = [ + c["id"] for c in engine.recall( + q["q"], + SearchFilter(workspace_id=wid), + k=k, + retrieval_profile=retrieval_profile, + ).chunks + ] else: ids = [i for i, _ in index.search(emb.embed([q["q"]])[0], k, filter=SearchFilter(workspace_id=wid))] @@ -89,7 +137,7 @@ def _arm_recall(dataset: list[dict], *, k: int, arm: str) -> float: wid = store.get_or_create_workspace("eval") rid = store.get_or_create_repo(wid, case.get("id", "c")) index = NumpyVectorIndex(store) - _seed_graph(store, workspace_id=wid, repo_id=rid, case=case) + entity_ids = _seed_graph(store, workspace_id=wid, repo_id=rid, case=case) mode = "1hop" if arm == "graph1hop" else "ppr" engine = RecallEngine(store, emb, index, IdentityReranker(), graph_mode=mode) tag_by_id = {} @@ -97,6 +145,14 @@ def _arm_recall(dataset: list[dict], *, k: int, arm: str) -> float: mid = store.add_memory(MemoryRecord( id="", content=m["text"], mtype=MemoryType.EPISODIC, scope=Scope.REPO, workspace_id=wid, repo_id=rid, embedding=emb.embed([m["text"]])[0])) + _link_fixture_mentions( + store, + memory_id=mid, + text=m["text"], + entity_ids=entity_ids, + workspace_id=wid, + repo_id=rid, + ) tag_by_id[mid] = m.get("tag") for q in case["questions"]: if arm == "vector": @@ -127,6 +183,13 @@ def main() -> None: print(f" vector arm : {_arm_recall(mh, k=5, arm='vector')}") print(f" graph 1-hop : {_arm_recall(mh, k=5, arm='graph1hop')} (reaches 1 hop only)") print(f" graph PPR : {_arm_recall(mh, k=5, arm='graphppr')} (multi-hop walk)") + print("\nEngraphis retrieval-policy fixture — recall@5") + print(f" balanced : {_score(mh, k=5, hybrid=True)}") + print( + " auto : " + f"{_score(mh, k=5, hybrid=True, retrieval_profile='auto')} " + "(opt-in graph specialization)" + ) if __name__ == "__main__": diff --git a/eval/benchmark.py b/eval/benchmark.py new file mode 100644 index 00000000..93615817 --- /dev/null +++ b/eval/benchmark.py @@ -0,0 +1,1034 @@ +"""Small, dependency-free primitives for reproducible public benchmark reports. + +This module intentionally owns only evaluation bookkeeping. It does not call a +model, download a dataset, or import a production backend, which keeps the +offline CI path reproducible. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import platform +import random +import sys +from copy import deepcopy +from pathlib import Path +from typing import Any, Callable, Iterable, Optional, Protocol, Sequence, Union + +from engraphis.core.textutil import estimate_tokens +from eval import metrics as retrieval_metrics + + +SCHEMA = "engraphis-benchmark/v2" +CANONICAL_TOKEN_BUDGETS = (256, 512, 1024, 2048, 4096) +CANONICAL_BASELINE_LABELS = ( + "no_retrieval", + "lexical_only", + "dense_only", + "dense_lexical_rrf", + "full_hybrid", + "full_history", + "no_graph", + "no_reranker", + "no_temporal_resolution", + "whole_document", +) +# Names come from the official harness. Revisions are immutable upstream commits +# resolved from the official GitHub/Hugging Face repositories on 2026-07-29. +LONGMEMEVAL_V2_CANONICAL_PROFILE_TEMPLATE = { + "benchmark": { + "repository": "xiaowu0162/LongMemEval-V2", + "repository_revision": "6f020ac2fc3275e46c706d3406e02c3ed79b7be2", + "dataset_revision": "f152293e235517d504809563c833d7190b8c713b", + }, + "reader": { + "model": "Qwen/Qwen3.5-9B", + "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + }, + "embedding": { + "model": "Qwen/Qwen3-Embedding-8B", + "revision": "1d8ad4ca9b3dd8059ad90a75d4983776a23d44af", + }, + "baseline_label": "REQUIRED", + "token_budgets": list(CANONICAL_TOKEN_BUDGETS), +} +_CANONICAL_COMMIT = "0123456789abcdef" +_CANONICAL_TOKEN_ACCOUNTING_METHOD = "pinned_reader_content_tokenizer" +_RANK_METRICS = tuple( + f"{metric}_at_{depth}" + for metric in ("recall", "mrr", "ndcg") + for depth in (1, 5, 10) +) +_GROUNDED_METRICS = ("grounded_f1", "abstention_f1") + + +class Tokenizer(Protocol): + """Minimal tokenizer contract accepted by :func:`count_tokens`.""" + + def encode(self, text: str) -> Sequence[Any]: + ... + + +def canonical_json(value: Any) -> str: + """Serialize config deterministically so its hash is portable.""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def sha256_file(path: Union[str, Path]) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_benchmark_config( + *, + run_label: str, + baseline_label: str, + token_budgets: Sequence[int] = CANONICAL_TOKEN_BUDGETS, + profile: Optional[dict] = None, +) -> dict: + """Build the labeled fixed-budget configuration required for a canonical run.""" + resolved_profile = deepcopy( + profile if profile is not None else LONGMEMEVAL_V2_CANONICAL_PROFILE_TEMPLATE + ) + resolved_profile["baseline_label"] = baseline_label + return { + "run_label": run_label, + "baseline_label": baseline_label, + "token_budgets": [int(budget) for budget in token_budgets], + "canonical_profile": resolved_profile, + } + + +def _sha256_error(value: Any, field: str, errors: list[str]) -> None: + if not isinstance(value, str) or len(value) != 64: + errors.append(f"{field} must be a 64-character SHA-256 hex string") + return + try: + int(value, 16) + except ValueError: + errors.append(f"{field} must be a 64-character SHA-256 hex string") + + +def _is_finite_number(value: Any) -> bool: + """Return whether ``value`` is a real finite number, excluding booleans.""" + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + ) + + +def _is_nonnegative_integer(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _reader_tokenizer_identity(profile: Any) -> Optional[str]: + """Return the only tokenizer identity valid for a canonical profile.""" + if not isinstance(profile, dict) or not isinstance(profile.get("reader"), dict): + return None + model = profile["reader"].get("model") + revision = profile["reader"].get("revision") + if not isinstance(model, str) or not isinstance(revision, str): + return None + return f"{model}@{revision}" + + +def _metric_matches(value: Any, expected: float) -> bool: + return _is_finite_number(value) and math.isclose( + float(value), float(expected), rel_tol=0.0, abs_tol=1e-6 + ) + + +def _rank_metrics_from_record(record: dict) -> Optional[dict[str, float]]: + """Recompute rank metrics only when the public evidence IDs are complete.""" + retrieved = record.get("retrieved_ids") + supporting = record.get("supporting_ids") + if ( + not isinstance(retrieved, list) + or not isinstance(supporting, list) + or not all(isinstance(item, str) for item in retrieved) + or not all(isinstance(item, str) for item in supporting) + ): + return None + return retrieval_metrics.retrieval_metrics_at_depths( + retrieved, supporting, depths=(1, 5, 10) + ) + + +def validate_canonical_profile(profile: Any) -> list[str]: + """Return validation errors for an immutable LongMemEval-V2 profile.""" + errors: list[str] = [] + if not isinstance(profile, dict): + return ["canonical_profile must be an object"] + required = ( + ("benchmark", "repository"), ("benchmark", "repository_revision"), + ("benchmark", "dataset_revision"), ("reader", "model"), ("reader", "revision"), + ("embedding", "model"), ("embedding", "revision"), + ) + for section, field in required: + value = profile.get(section, {}) if isinstance(profile.get(section), dict) else {} + item = value.get(field) + if not isinstance(item, str) or not item.strip() or item == "REQUIRED": + errors.append(f"canonical_profile.{section}.{field} is required") + elif field.endswith("revision") and ( + len(item) != 40 or any(char not in "0123456789abcdef" for char in item) + ): + errors.append( + f"canonical_profile.{section}.{field} must be an immutable 40-character commit" + ) + baseline = profile.get("baseline_label") + if baseline not in CANONICAL_BASELINE_LABELS: + errors.append("canonical_profile.baseline_label must name a declared baseline") + budgets = profile.get("token_budgets") + if budgets != list(CANONICAL_TOKEN_BUDGETS): + errors.append("canonical_profile.token_budgets must be the canonical fixed budgets") + return errors + + +def validate_report(report: Any, *, canonical: bool = False) -> list[str]: + """Deterministically validate a public ``engraphis-benchmark/v2`` envelope.""" + errors: list[str] = [] + if not isinstance(report, dict): + return ["report must be an object"] + if report.get("schema") != SCHEMA: + errors.append(f"schema must equal {SCHEMA}") + for field in ("suite", "system", "environment", "protocol", "metrics"): + if not isinstance(report.get(field), dict): + errors.append(f"{field} must be an object") + records = report.get("records") + exclusions = report.get("exclusions") + if not isinstance(records, list): + errors.append("records must be an array") + records = [] + if not isinstance(exclusions, list): + errors.append("exclusions must be an array") + exclusions = [] + suite = report.get("suite") if isinstance(report.get("suite"), dict) else {} + system = report.get("system") if isinstance(report.get("system"), dict) else {} + protocol = report.get("protocol") if isinstance(report.get("protocol"), dict) else {} + for field in ("name", "dataset"): + if not isinstance(suite.get(field), str) or not suite[field]: + errors.append(f"suite.{field} must be a non-empty string") + _sha256_error(suite.get("sha256"), "suite.sha256", errors) + if not isinstance(system.get("git_commit"), str) or not system["git_commit"]: + errors.append("system.git_commit must be a non-empty string") + elif canonical and ( + len(system["git_commit"]) != 40 + or any(char not in _CANONICAL_COMMIT for char in system["git_commit"]) + ): + errors.append("canonical system.git_commit must be an immutable lowercase 40-character commit") + declared_config_hash = system.get("config_sha256") + _sha256_error(declared_config_hash, "system.config_sha256", errors) + if not isinstance(protocol.get("config"), dict): + errors.append("protocol.config must be an object") + else: + actual_config_hash = sha256_text(canonical_json(protocol["config"])) + if declared_config_hash != actual_config_hash: + errors.append( + "system.config_sha256 must match the canonical protocol.config digest" + ) + record_ids: list[str] = [] + embedded_exclusions: dict[str, dict] = {} + for record in records: + if not isinstance(record, dict) or not isinstance(record.get("question_id"), str): + errors.append("each record requires a string question_id") + continue + question_id = record["question_id"] + if not question_id: + errors.append("each record question_id must be non-empty") + continue + record_ids.append(question_id) + embedded = record.get("excluded") + if embedded is not None: + if not isinstance(embedded, dict) or embedded.get("question_id") != question_id: + errors.append("each record exclusion must name that record question_id") + else: + embedded_exclusions[question_id] = embedded + + if len(record_ids) != len(set(record_ids)): + errors.append("record question_id values must be unique") + + exclusion_ids: list[str] = [] + for item in exclusions: + if not isinstance(item, dict) or not isinstance(item.get("question_id"), str): + errors.append("each exclusion requires a string question_id") + continue + question_id = item["question_id"] + if not question_id: + errors.append("each exclusion question_id must be non-empty") + continue + exclusion_ids.append(question_id) + if question_id not in record_ids: + errors.append("each exclusion must name a reported question_id") + elif embedded_exclusions.get(question_id) != item: + errors.append("top-level exclusions must exactly match per-record exclusions") + if len(exclusion_ids) != len(set(exclusion_ids)): + errors.append("exclusion question_id values must be unique") + if set(exclusion_ids) != set(embedded_exclusions): + errors.append("top-level exclusions must exactly match per-record exclusions") + + if ( + not _is_nonnegative_integer(protocol.get("n_total")) + or protocol.get("n_total") != len(records) + ): + errors.append("protocol.n_total must equal records length") + if ( + not _is_nonnegative_integer(protocol.get("n_scored")) + or protocol.get("n_scored") != len(records) - len(embedded_exclusions) + ): + errors.append("protocol.n_scored must equal records minus exclusions") + if canonical: + config = protocol.get("config") if isinstance(protocol.get("config"), dict) else {} + if protocol.get("complete_dataset") is not True: + errors.append("canonical protocol.complete_dataset must be true") + source_questions = protocol.get("source_questions") + if not _is_nonnegative_integer(source_questions) or source_questions == 0: + errors.append("canonical protocol.source_questions must be a positive integer") + elif source_questions != len(records) or source_questions != protocol.get("n_total"): + errors.append( + "canonical protocol.source_questions must equal protocol.n_total and records length" + ) + if config.get("baseline_label") not in CANONICAL_BASELINE_LABELS: + errors.append("protocol.config.baseline_label must name a declared baseline") + if config.get("token_budgets") != list(CANONICAL_TOKEN_BUDGETS): + errors.append("protocol.config.token_budgets must be the canonical fixed budgets") + configured_budget = config.get("token_budget") + if configured_budget is not None and not _is_nonnegative_integer(configured_budget): + errors.append("canonical protocol.config.token_budget must be a non-negative integer or null") + profile = config.get("canonical_profile") + errors.extend(validate_canonical_profile(profile)) + if isinstance(profile, dict) and profile.get("baseline_label") != config.get( + "baseline_label" + ): + errors.append( + "protocol.config.baseline_label must match canonical_profile.baseline_label" + ) + _validate_canonical_measurement_contract(report, records, profile, errors) + return errors + + +def _validate_canonical_measurement_contract( + report: dict, + records: Sequence[dict], + profile: Any, + errors: list[str], +) -> None: + """Require the per-question evidence that makes a canonical result auditable.""" + models = report.get("models") + embedder = models.get("embedder") if isinstance(models, dict) else None + expected_embedding = profile.get("embedding", {}) if isinstance(profile, dict) else {} + if not isinstance(embedder, dict): + errors.append("canonical reports require models.embedder provenance") + else: + for field in ("name", "model_id", "revision", "sha256"): + if not isinstance(embedder.get(field), str) or not embedder[field]: + errors.append(f"canonical models.embedder.{field} is required") + _sha256_error(embedder.get("sha256"), "canonical models.embedder.sha256", errors) + if embedder.get("model_id") != expected_embedding.get("model"): + errors.append("canonical models.embedder.model_id must match canonical_profile.embedding.model") + if embedder.get("revision") != expected_embedding.get("revision"): + errors.append("canonical models.embedder.revision must match canonical_profile.embedding.revision") + if not isinstance(report.get("metrics"), dict): + return + metrics = report["metrics"] + expected_tokenizer_identity = _reader_tokenizer_identity(profile) + configured_budget = ( + report.get("protocol", {}).get("config", {}).get("token_budget") + if isinstance(report.get("protocol"), dict) + and isinstance(report.get("protocol", {}).get("config"), dict) + else None + ) + for field in _RANK_METRICS: + value = metrics.get(field) + if not _is_finite_number(value) or not 0.0 <= float(value) <= 1.0: + errors.append(f"canonical metrics.{field} must be a number in [0, 1]") + confidence = metrics.get("confidence_intervals") + _validate_confidence_intervals(confidence, metrics, records, errors) + paired = metrics.get("paired_bootstrap") + _validate_paired_bootstrap(paired, records, errors) + _validate_grounded_metric_availability(metrics, records, errors) + _validate_fixed_budget_curve( + metrics, records, expected_tokenizer_identity, errors + ) + for record in records: + if not isinstance(record, dict): + continue + if "q" in record: + errors.append("canonical records must not contain raw query text") + query_hash = record.get("question_sha256") + _sha256_error(query_hash, "canonical record.question_sha256", errors) + latency = record.get("latency_ms") + if not _is_finite_number(latency) or float(latency) < 0: + errors.append("canonical records require non-negative latency_ms") + context_tokens = record.get("context_tokens") + if not _is_finite_number(context_tokens) or float(context_tokens) < 0: + errors.append("canonical records require non-negative finite context_tokens") + elif ( + _is_nonnegative_integer(configured_budget) + and float(context_tokens) > configured_budget + ): + errors.append("canonical record context_tokens must not exceed protocol token_budget") + usage = record.get("usage") + if isinstance(usage, dict): + usage_budget = usage.get("budget_tokens") + usage_context = usage.get("context_tokens") + if not _is_finite_number(usage_budget) or float(usage_budget) < 0: + errors.append("canonical record usage.budget_tokens must be non-negative and finite") + if not _is_finite_number(usage_context) or float(usage_context) < 0: + errors.append("canonical record usage.context_tokens must be non-negative and finite") + elif _is_finite_number(usage_budget) and float(usage_context) > float(usage_budget): + errors.append("canonical record usage.context_tokens must not exceed usage.budget_tokens") + if ( + _is_finite_number(context_tokens) + and _is_finite_number(usage_context) + and float(context_tokens) != float(usage_context) + ): + errors.append("canonical record context_tokens must equal usage.context_tokens") + if ( + _is_nonnegative_integer(configured_budget) + and _is_finite_number(usage_budget) + and float(usage_budget) != configured_budget + ): + errors.append("canonical record usage.budget_tokens must equal protocol token_budget") + for field in ("source_tokens", "saved_tokens"): + value = usage.get(field) + if field in usage and ( + not _is_finite_number(value) or float(value) < 0 + ): + errors.append( + f"canonical record usage.{field} must be non-negative and finite" + ) + savings_ratio = usage.get("savings_ratio") + if "savings_ratio" in usage and ( + not _is_finite_number(savings_ratio) + or not 0.0 <= float(savings_ratio) <= 1.0 + ): + errors.append("canonical record usage.savings_ratio must be a number in [0, 1]") + for field in ("packed_count", "omitted_count"): + if field in usage and not _is_nonnegative_integer(usage.get(field)): + errors.append( + f"canonical record usage.{field} must be a non-negative integer" + ) + if usage.get("token_counter") != expected_tokenizer_identity: + errors.append( + "canonical record usage.token_counter must match canonical_profile.reader" + ) + else: + errors.append( + "canonical records require usage with pinned reader token accounting" + ) + for field in ("answerable", "grounded", "abstained"): + if field in record and not isinstance(record.get(field), bool): + errors.append(f"canonical record {field} must be boolean when present") + method = record.get("context_token_method") + if method != _CANONICAL_TOKEN_ACCOUNTING_METHOD: + errors.append( + "canonical records require " + "context_token_method=pinned_reader_content_tokenizer" + ) + if record.get("context_tokenizer_identity") != expected_tokenizer_identity: + errors.append( + "canonical record context_tokenizer_identity must match canonical_profile.reader" + ) + recomputed = _rank_metrics_from_record(record) + if recomputed is None: + errors.append( + "canonical records require string-array retrieved_ids and supporting_ids" + ) + for field in _RANK_METRICS: + value = record.get(field) + if not _is_finite_number(value) or not 0.0 <= float(value) <= 1.0: + errors.append(f"canonical records require {field} in [0, 1]") + elif recomputed is not None and not _metric_matches(value, recomputed[field]): + errors.append( + f"canonical record {field} must match retrieved_ids and supporting_ids" + ) + _validate_rank_metric_aggregates(metrics, records, errors) + + +def _validate_rank_metric_aggregates( + metrics: dict, records: Sequence[dict], errors: list[str] +) -> None: + """Recompute every canonical aggregate from non-excluded question evidence.""" + scored = [ + record for record in records + if isinstance(record, dict) and not record.get("excluded") + ] + recomputed = [_rank_metrics_from_record(record) for record in scored] + if any(item is None for item in recomputed): + return + for field in _RANK_METRICS: + expected = ( + sum(item[field] for item in recomputed if item is not None) / len(recomputed) + if recomputed else 0.0 + ) + if not _metric_matches(metrics.get(field), expected): + errors.append( + f"canonical metrics.{field} must equal the non-excluded record mean" + ) + + +def _validate_confidence_intervals( + confidence: Any, + metrics: dict, + records: Sequence[dict], + errors: list[str], +) -> None: + """Require complete, bounded confidence intervals tied to reported point estimates.""" + if not isinstance(confidence, dict) or set(confidence) != set(_RANK_METRICS): + errors.append( + "canonical metrics.confidence_intervals must exactly cover every rank metric" + ) + return + expected_keys = { + "point", "low", "high", "n", "seed", "iterations", "strata_key", + } + n_scored = sum( + 1 for record in records + if isinstance(record, dict) and not record.get("excluded") + ) + for field in _RANK_METRICS: + interval = confidence[field] + prefix = f"canonical metrics.confidence_intervals.{field}" + if not isinstance(interval, dict) or set(interval) != expected_keys: + errors.append(f"{prefix} must match the canonical confidence interval schema") + continue + point = interval.get("point") + low = interval.get("low") + high = interval.get("high") + if not all( + _is_finite_number(value) and 0.0 <= float(value) <= 1.0 + for value in (point, low, high) + ): + errors.append(f"{prefix} point/low/high must be finite numbers in [0, 1]") + elif not float(low) <= float(point) <= float(high): + errors.append(f"{prefix} must satisfy low <= point <= high") + aggregate = metrics.get(field) + if ( + not _is_finite_number(aggregate) + or not _metric_matches(point, float(aggregate)) + ): + errors.append(f"{prefix}.point must match metrics.{field}") + if not _is_nonnegative_integer(interval.get("n")) or interval["n"] != n_scored: + errors.append(f"{prefix}.n must equal the non-excluded record count") + if not _is_nonnegative_integer(interval.get("seed")): + errors.append(f"{prefix}.seed must be a non-negative integer") + iterations = interval.get("iterations") + if not _is_nonnegative_integer(iterations) or iterations == 0: + errors.append(f"{prefix}.iterations must be a positive integer") + if interval.get("strata_key") != "category": + errors.append(f"{prefix}.strata_key must equal category") + + +def _validate_paired_bootstrap( + paired: Any, records: Sequence[dict], errors: list[str] +) -> None: + """Validate exact available/unavailable paired-bootstrap payload shapes.""" + prefix = "canonical metrics.paired_bootstrap" + if not isinstance(paired, dict) or not isinstance(paired.get("available"), bool): + errors.append(f"{prefix} must explicitly state availability") + return + n_scored = sum( + 1 for record in records + if isinstance(record, dict) and not record.get("excluded") + ) + if paired["available"] is False: + expected_keys = { + "available", "reason", "n", "delta", "low", "high", "iterations", + } + if set(paired) != expected_keys: + errors.append(f"{prefix} unavailable payload must match the canonical schema") + return + if not isinstance(paired.get("reason"), str) or not paired["reason"].strip(): + errors.append(f"{prefix}.reason must be a non-empty string when unavailable") + if paired.get("n") != 0 or isinstance(paired.get("n"), bool): + errors.append(f"{prefix}.n must be zero when unavailable") + if any(paired.get(field) is not None for field in ("delta", "low", "high")): + errors.append(f"{prefix} delta/low/high must be null when unavailable") + iterations = paired.get("iterations") + if not _is_nonnegative_integer(iterations) or iterations == 0: + errors.append(f"{prefix}.iterations must be a positive integer") + return + + expected_keys = { + "available", "metric", "delta", "low", "high", "n", "seed", "iterations", + } + if set(paired) != expected_keys: + errors.append(f"{prefix} available payload must match the canonical schema") + return + if paired.get("metric") not in _RANK_METRICS: + errors.append(f"{prefix}.metric must name a canonical rank metric") + delta = paired.get("delta") + low = paired.get("low") + high = paired.get("high") + if not all( + _is_finite_number(value) and -1.0 <= float(value) <= 1.0 + for value in (delta, low, high) + ): + errors.append(f"{prefix} delta/low/high must be finite numbers in [-1, 1]") + elif not float(low) <= float(delta) <= float(high): + errors.append(f"{prefix} must satisfy low <= delta <= high") + if ( + not _is_nonnegative_integer(paired.get("n")) + or paired["n"] == 0 + or paired["n"] != n_scored + ): + errors.append(f"{prefix}.n must equal the positive non-excluded record count") + if not _is_nonnegative_integer(paired.get("seed")): + errors.append(f"{prefix}.seed must be a non-negative integer") + iterations = paired.get("iterations") + if not _is_nonnegative_integer(iterations) or iterations == 0: + errors.append(f"{prefix}.iterations must be a positive integer") + + +def _validate_grounded_metric_availability( + metrics: dict, records: Sequence[dict], errors: list[str] +) -> None: + """Require measured F1 or an explicit, machine-readable unavailable state.""" + specs = { + "grounded_f1": ("grounded", retrieval_metrics.grounded_precision_recall_f1), + "abstention_f1": ("abstained", retrieval_metrics.abstention_precision_recall_f1), + } + labeled = [ + record for record in records + if isinstance(record, dict) and isinstance(record.get("answerable"), bool) + ] + for field, (prediction_field, score) in specs.items(): + value = metrics.get(field) + if _is_finite_number(value): + if not 0.0 <= float(value) <= 1.0: + errors.append(f"canonical metrics.{field} must be a number in [0, 1]") + continue + if not labeled or not all( + isinstance(record.get(prediction_field), bool) for record in labeled + ): + errors.append( + f"canonical metrics.{field} requires labeled per-question " + f"{prediction_field} values; otherwise use an unavailable reason" + ) + continue + expected = score( + [record[prediction_field] for record in labeled], + [record["answerable"] for record in labeled], + ) + if not _metric_matches(value, float(expected["f1"])): + errors.append( + f"canonical metrics.{field} must be recomputed from per-question labels" + ) + summary_name = field.removesuffix("_f1") + summary = metrics.get(summary_name) + if not isinstance(summary, dict) or summary.get("available") is not True: + errors.append( + f"canonical numeric metrics.{field} requires an available " + f"metrics.{summary_name} count summary" + ) + continue + for summary_field, expected_value in expected.items(): + reported = summary.get(summary_field) + matches = ( + reported == expected_value + and not isinstance(reported, bool) + if isinstance(expected_value, int) + else _metric_matches(reported, float(expected_value)) + ) + if not matches: + errors.append( + f"canonical metrics.{summary_name}.{summary_field} must be " + "recomputed from per-question labels" + ) + continue + if ( + isinstance(value, dict) + and value.get("available") is False + and isinstance(value.get("reason"), str) + and value["reason"].strip() + ): + continue + errors.append( + f"canonical metrics.{field} must be a number in [0, 1] or an unavailable reason" + ) + + +def _validate_fixed_budget_curve( + metrics: dict, + records: Sequence[dict], + expected_tokenizer_identity: Optional[str], + errors: list[str], +) -> None: + """Require measured, per-question evidence at every canonical token budget. + + Merely declaring budgets in ``protocol.config`` does not show that retrieval was + actually run at those budgets. The curve therefore contains its own per-question + rows, whose IDs and exclusion state must exactly match the report's public record + set. An unavailable curve is retained as a machine-readable status but cannot + qualify as canonical evidence. + """ + curve = metrics.get("fixed_budget_curve") + if not isinstance(curve, dict): + errors.append("canonical metrics.fixed_budget_curve must be an object") + return + if curve.get("available") is False: + if not isinstance(curve.get("reason"), str) or not curve["reason"].strip(): + errors.append("canonical metrics.fixed_budget_curve unavailable state requires a reason") + errors.append("canonical fixed-budget curve is unavailable and cannot qualify as evidence") + return + if curve.get("available") is not True: + errors.append("canonical metrics.fixed_budget_curve must explicitly state availability") + return + rows = curve.get("rows") + if not isinstance(rows, list): + errors.append("canonical metrics.fixed_budget_curve.rows must be an array") + return + expected_ids = {record.get("question_id") for record in records if isinstance(record, dict)} + expected_exclusions = { + record.get("question_id"): bool(record.get("excluded")) + for record in records + if isinstance(record, dict) + } + by_budget: dict[int, dict] = {} + for row in rows: + if not isinstance(row, dict) or not _is_nonnegative_integer(row.get("token_budget")): + errors.append("canonical fixed-budget curve rows require an integer token_budget") + continue + budget = row["token_budget"] + if budget in by_budget: + errors.append("canonical fixed-budget curve token_budget values must be unique") + continue + by_budget[budget] = row + if set(by_budget) != set(CANONICAL_TOKEN_BUDGETS): + errors.append("canonical fixed-budget curve must contain every canonical token budget") + for budget in CANONICAL_TOKEN_BUDGETS: + row = by_budget.get(budget) + if row is None: + continue + if row.get("status") != "measured": + errors.append(f"canonical fixed-budget curve {budget} must be measured") + for field in _RANK_METRICS: + value = row.get(field) + if not _is_finite_number(value) or not 0.0 <= float(value) <= 1.0: + errors.append( + f"canonical fixed-budget curve {budget} requires {field} in [0, 1]" + ) + measurements = row.get("records") + if not isinstance(measurements, list): + errors.append(f"canonical fixed-budget curve {budget} requires per-question records") + continue + measurement_ids = [] + scored_metrics: list[dict[str, float]] = [] + for item in measurements: + if not isinstance(item, dict) or not isinstance(item.get("question_id"), str): + errors.append(f"canonical fixed-budget curve {budget} records require question_id") + continue + measurement_ids.append(item["question_id"]) + if bool(item.get("excluded")) != expected_exclusions.get(item["question_id"]): + errors.append( + f"canonical fixed-budget curve {budget} records must preserve exclusion state" + ) + context_tokens = item.get("context_tokens") + if ( + not _is_finite_number(context_tokens) + or not 0 <= float(context_tokens) <= budget + ): + errors.append( + f"canonical fixed-budget curve {budget} records require context_tokens within budget" + ) + if item.get("context_token_method") != _CANONICAL_TOKEN_ACCOUNTING_METHOD: + errors.append( + f"canonical fixed-budget curve {budget} records require " + "context_token_method=pinned_reader_content_tokenizer" + ) + if item.get("context_tokenizer_identity") != expected_tokenizer_identity: + errors.append( + f"canonical fixed-budget curve {budget} record tokenizer identity " + "must match canonical_profile.reader" + ) + recomputed = _rank_metrics_from_record(item) + if recomputed is None: + errors.append( + f"canonical fixed-budget curve {budget} records require string-array " + "retrieved_ids and supporting_ids" + ) + elif not item.get("excluded"): + scored_metrics.append(recomputed) + for field in _RANK_METRICS: + value = item.get(field) + if ( + not _is_finite_number(value) + or not 0.0 <= float(value) <= 1.0 + ): + errors.append( + f"canonical fixed-budget curve {budget} records require {field} in [0, 1]" + ) + elif recomputed is not None and not _metric_matches(value, recomputed[field]): + errors.append( + f"canonical fixed-budget curve {budget} record {field} must match " + "retrieved_ids and supporting_ids" + ) + if len(measurement_ids) != len(set(measurement_ids)) or set(measurement_ids) != expected_ids: + errors.append( + f"canonical fixed-budget curve {budget} records must exactly cover report question_ids" + ) + if not _is_nonnegative_integer(row.get("n_total")) or row.get("n_total") != len(records): + errors.append(f"canonical fixed-budget curve {budget} n_total must equal report records") + scored = sum(1 for item in measurements if isinstance(item, dict) and not item.get("excluded")) + if not _is_nonnegative_integer(row.get("n_scored")) or row.get("n_scored") != scored: + errors.append(f"canonical fixed-budget curve {budget} n_scored must match records") + if len(scored_metrics) == scored: + for field in _RANK_METRICS: + expected = ( + sum(item[field] for item in scored_metrics) / len(scored_metrics) + if scored_metrics else 0.0 + ) + if not _metric_matches(row.get(field), expected): + errors.append( + f"canonical fixed-budget curve {budget} {field} must equal " + "the non-excluded record mean" + ) + + +def write_canonical_artifact(report: dict, output: Union[str, Path], *, canonical: bool = False) -> dict: + """Write immutable canonical JSON and a SHA-256 sidecar after validation. + + Repeating an identical write is harmless. A different payload at the same + path is rejected, avoiding accidental replacement of a public evidence run. + """ + errors = validate_report(report, canonical=canonical) + if errors: + raise ValueError("invalid benchmark report: " + "; ".join(errors)) + payload = canonical_json(report).encode("utf-8") + b"\n" + digest = hashlib.sha256(payload).hexdigest() + artifact = Path(output) + if artifact.exists() and artifact.read_bytes() != payload: + raise FileExistsError(f"refusing to replace immutable artifact: {artifact}") + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(payload) + sidecar = artifact.with_name(artifact.name + ".sha256") + checksum = f"{digest} {artifact.name}\n".encode("ascii") + if sidecar.exists() and sidecar.read_bytes() != checksum: + raise FileExistsError(f"refusing to replace immutable checksum: {sidecar}") + sidecar.write_bytes(checksum) + return {"artifact": str(artifact), "sha256": digest, "checksum": str(sidecar)} + + +def count_tokens( + text: str, tokenizer: Optional[Union[Tokenizer, Callable[[str], int]]] = None +) -> dict: + """Count tokens with an injected exact tokenizer or deterministic fallback. + + The caller must supply the tokenizer used by its reader to call this count + ``exact``. The fallback is intentionally labelled an estimate rather than + pretending a whitespace heuristic is an LLM tokenizer. + """ + if tokenizer is None: + return {"tokens": estimate_tokens(text), "method": "deterministic_estimate"} + if callable(tokenizer) and not hasattr(tokenizer, "encode"): + return {"tokens": int(tokenizer(text)), "method": "injected"} + return {"tokens": len(tokenizer.encode(text)), "method": "injected"} + + +def packed_context_tokens( + chunks: Iterable[str], + *, + tokenizer: Optional[Union[Tokenizer, Callable[[str], int]]] = None, +) -> dict: + """Count the exact injected context, including chunk separators.""" + return count_tokens("\n\n".join(chunk for chunk in chunks if chunk), tokenizer) + + +def exclusion(question_id: str, reason: str, *, detail: str = "") -> dict: + return {"question_id": question_id, "reason": reason, "detail": detail} + + +def question_record( + question_id: str, + *, + category: str = "unknown", + retrieved_ids: Optional[Sequence[str]] = None, + supporting_ids: Optional[Sequence[str]] = None, + context_tokens: Optional[int] = None, + latency_ms: Optional[float] = None, + abstained: Optional[bool] = None, + excluded: Optional[dict] = None, + **metrics: Any, +) -> dict: + """Create a stable per-question public record without storing raw corpora.""" + record: dict[str, Any] = { + "question_id": question_id, + "category": category, + "retrieved_ids": list(retrieved_ids or []), + "supporting_ids": list(supporting_ids or []), + } + if context_tokens is not None: + record["context_tokens"] = int(context_tokens) + if latency_ms is not None: + record["latency_ms"] = round(float(latency_ms), 6) + if abstained is not None: + record["abstained"] = bool(abstained) + if excluded is not None: + record["excluded"] = excluded + record.update(metrics) + return record + + +def _mean(values: Sequence[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +def stratified_bootstrap_ci( + records: Sequence[dict], + metric: Callable[[Sequence[dict]], float], + *, + strata_key: str = "category", + iterations: int = 1000, + seed: int = 20260729, + alpha: float = 0.05, +) -> dict: + """Deterministic percentile CI that resamples within each named stratum.""" + usable = [record for record in records if not record.get("excluded")] + groups: dict[str, list[dict]] = {} + for record in usable: + groups.setdefault(str(record.get(strata_key, "unknown")), []).append(record) + point = metric(usable) + if not usable or iterations <= 0: + return {"point": point, "low": point, "high": point, "n": len(usable), "seed": seed} + rng = random.Random(seed) + samples: list[float] = [] + for _ in range(iterations): + sampled = [item for group in groups.values() for item in + (rng.choice(group) for _ in range(len(group)))] + samples.append(metric(sampled)) + samples.sort() + low_index = max(0, math.floor((alpha / 2) * (iterations - 1))) + high_index = min(iterations - 1, math.ceil((1 - alpha / 2) * (iterations - 1))) + return { + "point": round(point, 6), "low": round(samples[low_index], 6), + "high": round(samples[high_index], 6), "n": len(usable), "seed": seed, + "iterations": iterations, "strata_key": strata_key, + } + + +def paired_bootstrap_ci( + pairs: Sequence[tuple[float, float]], + *, + iterations: int = 1000, + seed: int = 20260729, + alpha: float = 0.05, +) -> dict: + """CI for mean(candidate - baseline) over paired benchmark observations.""" + deltas = [candidate - baseline for candidate, baseline in pairs] + point = _mean(deltas) + if not deltas or iterations <= 0: + return {"delta": point, "low": point, "high": point, "n": len(deltas), "seed": seed} + rng = random.Random(seed) + sampled = [] + for _ in range(iterations): + sampled.append(_mean([rng.choice(deltas) for _ in deltas])) + sampled.sort() + low_index = max(0, math.floor((alpha / 2) * (iterations - 1))) + high_index = min(iterations - 1, math.ceil((1 - alpha / 2) * (iterations - 1))) + return { + "delta": round(point, 6), "low": round(sampled[low_index], 6), + "high": round(sampled[high_index], 6), "n": len(deltas), "seed": seed, + "iterations": iterations, + } + + +def fixed_budget_curve(records: Sequence[dict], budgets: Sequence[int]) -> list[dict]: + """Summarize evidence quality available at each packed-context token budget. + + Each record has ordered ``chunks`` of ``{"id", "tokens"}`` and + ``supporting_ids``. This records only retrieval/capping behavior; callers + can attach reader quality separately. + """ + result = [] + usable = [record for record in records if not record.get("excluded")] + for budget in sorted(set(int(value) for value in budgets if value >= 0)): + recalls, hits, used_tokens = [], [], [] + for record in usable: + used = 0 + ids = [] + for chunk in record.get("chunks", []): + tokens = int(chunk.get("tokens", 0)) + if used + tokens > budget: + continue + used += tokens + ids.append(str(chunk.get("id", ""))) + supporting = set(str(value) for value in record.get("supporting_ids", [])) + overlap = len(supporting.intersection(ids)) + recalls.append(overlap / len(supporting) if supporting else 1.0) + hits.append(1.0 if overlap else 0.0) + used_tokens.append(used) + result.append({ + "token_budget": budget, "n": len(usable), "recall": round(_mean(recalls), 6), + "hit_rate": round(_mean(hits), 6), "mean_packed_tokens": round(_mean(used_tokens), 3), + }) + return result + + +def report_envelope( + *, + suite: str, + dataset_path: Union[str, Path], + config: dict, + records: Sequence[dict], + metrics: Optional[dict] = None, + exclusions: Optional[Sequence[dict]] = None, + git_commit: str = "unknown", +) -> dict: + """Build a JSON-safe, provenance-complete public benchmark envelope.""" + path = Path(dataset_path) + resolved_exclusions = list(exclusions or []) + resolved_exclusions.extend(record["excluded"] for record in records if record.get("excluded")) + # An adapter may supply both top-level and per-record exclusions. Retain one + # canonical representation so ``n_scored`` remains an honest denominator. + unique_exclusions = [] + seen_exclusions = set() + for item in resolved_exclusions: + marker = canonical_json(item) + if marker not in seen_exclusions: + unique_exclusions.append(item) + seen_exclusions.add(marker) + return { + "schema": SCHEMA, + "suite": {"name": suite, "dataset": path.name, "sha256": sha256_file(path)}, + "system": {"git_commit": git_commit, "config_sha256": sha256_text(canonical_json(config))}, + "environment": { + "python": sys.version.split()[0], "platform": platform.platform(), + }, + "protocol": {"config": config, "n_total": len(records), + "n_scored": len(records) - len(unique_exclusions)}, + "metrics": metrics or {}, "exclusions": unique_exclusions, + "records": list(records), + } + + +def main(argv: Optional[list[str]] = None) -> int: + """Validate an existing report and write its immutable canonical artifact.""" + parser = argparse.ArgumentParser(description="Validate and write an Engraphis benchmark artifact.") + parser.add_argument("--input", required=True, help="JSON report envelope to validate.") + parser.add_argument("--output", required=True, help="Canonical JSON artifact path.") + parser.add_argument( + "--canonical", action="store_true", help="Require the pinned LongMemEval-V2 profile." + ) + args = parser.parse_args(argv) + try: + report = json.loads(Path(args.input).read_text(encoding="utf-8")) + written = write_canonical_artifact(report, args.output, canonical=args.canonical) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"benchmark artifact error: {exc}", file=sys.stderr) + return 2 + print(canonical_json(written)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/configs/longmemeval_v2_engraphis.json b/eval/configs/longmemeval_v2_engraphis.json new file mode 100644 index 00000000..ff69028b --- /dev/null +++ b/eval/configs/longmemeval_v2_engraphis.json @@ -0,0 +1,15 @@ +{ + "memory_type": "engraphis", + "memory_params": { + "context_k": 8, + "max_context_tokens": 1024, + "require_exact_reader_tokenizer": true, + "reader_tokenizer_model": "Qwen/Qwen3.5-9B", + "reader_tokenizer_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "tokenizer_identity": "Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "retrieval_profile": "balanced", + "embed_model": "Qwen/Qwen3-Embedding-8B", + "embed_revision": "1d8ad4ca9b3dd8059ad90a75d4983776a23d44af", + "vector_backend": "numpy" + } +} diff --git a/eval/datasets/adversarial.jsonl b/eval/datasets/adversarial.jsonl new file mode 100644 index 00000000..f4fbc08c --- /dev/null +++ b/eval/datasets/adversarial.jsonl @@ -0,0 +1 @@ +{"id":"adversarial-grounding","memories":[{"tag":"auth","text":"The API uses PASETO v4 public tokens for authentication."},{"tag":"noise","text":"The office kitchen orders sourdough every Friday."}],"questions":[{"id":"auth-answerable","category":"supported","q":"Which token format authenticates the API?","answer":"PASETO v4","supporting":["auth"],"answerable":true},{"id":"sourdough-off-topic","category":"off-topic","q":"How do I bake sourdough bread?","answer":"","supporting":[],"answerable":false,"exclusion_reason":"off_topic_no_gold_evidence"}]} diff --git a/eval/external.py b/eval/external.py index ac487c01..89638063 100644 --- a/eval/external.py +++ b/eval/external.py @@ -1,16 +1,12 @@ """External benchmark adapter — run LoCoMo / LongMemEval through the real engine. The fixture evals (``eval.harness`` on ``sample.jsonl``/``codemem.jsonl``) are a -pipeline-correctness gate, not a performance claim. This adapter loads the two -benchmarks the field actually quotes and pushes them through the *same* -``MemoryEngine`` write path (conflict resolution, evolution) and hybrid recall that -ships — so the number you get is about the product, not a bare index. +pipeline-correctness gate, not a public benchmark claim. This adapter loads each +benchmark and pushes it through the shipped ``MemoryEngine`` write path (conflict +resolution, evolution) and hybrid recall. -What it measures — honestly: **retrieval** (evidence recall@k / hit@k), not -end-to-end QA accuracy. Published LoCoMo/LongMemEval scores from other systems also hinge -on an answering LLM + judge; this harness isolates the part Engraphis owns, needs no -API key, and states exactly that in the report. Add an answering model on top for a -QA-accuracy number when you want one. +It measures **retrieval** (evidence recall@k / hit@k), not end-to-end QA accuracy. +An official answering model and evaluator are required before reporting QA accuracy. Usage:: @@ -24,6 +20,9 @@ # Plumbing check without the model download (deterministic embedder): python -m eval.external --dataset locomo10.json --format locomo --offline --limit 2 + # A canonical run refuses --limit so its denominator cannot be partial: + python -m eval.external --dataset longmemeval_s.json --format longmemeval --canonical + Both loaders normalize to the ``eval.harness`` case shape, so every metric and resolution behaviour is identical to the CI gate. """ @@ -31,6 +30,7 @@ import argparse import json +import sys import time from pathlib import Path from typing import Optional @@ -46,8 +46,9 @@ def load_locomo(path: str, *, limit: Optional[int] = None) -> list[dict]: Each dialog turn becomes one memory tagged with its LoCoMo ``dia_id`` (e.g. ``D1:3``); each QA item's ``evidence`` lists the supporting ``dia_id``s. - Adversarial items (category 5) have no evidence and are skipped — retrieval - recall is undefined for "unanswerable". + Adversarial items (category 5) are retained with ``answerable=False``. A + retrieval score is undefined for those items, but retaining them prevents a + public report from silently changing the benchmark denominator. """ raw = json.loads(Path(path).read_text(encoding="utf-8")) if isinstance(raw, dict): @@ -71,13 +72,18 @@ def load_locomo(path: str, *, limit: Optional[int] = None) -> list[dict]: prefix = f"[{stamp}] " if stamp else "" memories.append({"tag": tag, "text": f"{prefix}{speaker}: {text}"}) questions = [] - for qa in sample.get("qa") or []: + for question_number, qa in enumerate(sample.get("qa") or []): supporting = [str(e).strip() for e in (qa.get("evidence") or []) if str(e).strip()] - if not supporting: - continue - questions.append({"q": str(qa.get("question") or ""), - "answer": str(qa.get("answer") or ""), - "supporting": supporting}) + category = str(qa.get("category") or "unknown") + questions.append({ + "id": f"{sample.get('sample_id') or len(cases)}:{question_number}", + "q": str(qa.get("question") or ""), + "answer": str(qa.get("answer") or ""), + "supporting": supporting, + "category": category, + "answerable": bool(supporting), + "exclusion_reason": "no_gold_evidence" if not supporting else "", + }) if memories and questions: cases.append({"id": str(sample.get("sample_id") or f"locomo-{len(cases)}"), "memories": memories, "questions": questions}) @@ -91,15 +97,13 @@ def load_longmemeval(path: str, *, limit: Optional[int] = None) -> list[dict]: Each haystack *session* becomes one memory (turns joined, newline-separated), tagged with its session id; ``answer_session_ids`` are the supporting evidence. - Abstention instances (id ending ``_abs``) are skipped — same rationale as - LoCoMo's adversarial category. + Abstention instances (id ending ``_abs``) are retained with their question + type and an explicit ``answerable=False`` marker. """ raw = json.loads(Path(path).read_text(encoding="utf-8")) cases = [] for inst in raw[: limit or len(raw)]: qid = str(inst.get("question_id") or f"lme-{len(cases)}") - if qid.endswith("_abs"): - continue session_ids = inst.get("haystack_session_ids") or [] sessions = inst.get("haystack_sessions") or [] dates = inst.get("haystack_dates") or [""] * len(sessions) @@ -114,24 +118,43 @@ def load_longmemeval(path: str, *, limit: Optional[int] = None) -> list[dict]: prefix = f"[{date}] " if date else "" memories.append({"tag": str(sid), "text": prefix + "\n".join(lines)}) supporting = [str(s) for s in (inst.get("answer_session_ids") or [])] - if memories and supporting: + if memories: cases.append({"id": qid, "memories": memories, "questions": [{"q": str(inst.get("question") or ""), "answer": str(inst.get("answer") or ""), - "supporting": supporting}]}) + "supporting": supporting, + "id": qid, + "category": ("abstention" if qid.endswith("_abs") + else str(inst.get("question_type") or "unknown")), + "answerable": not qid.endswith("_abs"), + "question_date": str(inst.get("question_date") or ""), + "exclusion_reason": ( + "abstention_no_gold_evidence" + if qid.endswith("_abs") else "" + )}]}) return cases LOADERS = {"locomo": load_locomo, "longmemeval": load_longmemeval} -def main() -> int: +def source_case_count(path: str) -> int: + """Count source cases before normalization so canonical runs catch drops.""" + raw = json.loads(Path(path).read_text(encoding="utf-8")) + return 1 if isinstance(raw, dict) else len(raw) if isinstance(raw, list) else 0 + + +def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser(description="Run an external memory benchmark through Engraphis.") ap.add_argument("--dataset", required=True, help="Path to the benchmark JSON file.") ap.add_argument("--format", required=True, choices=sorted(LOADERS), help="Benchmark format.") ap.add_argument("--k", type=int, default=10) ap.add_argument("--limit", type=int, default=None, help="Cap the number of cases.") + ap.add_argument( + "--canonical", action="store_true", + help="Require a full official-dataset run; rejects --limit/partial input.", + ) ap.add_argument("--embed-model", default="sentence-transformers/all-MiniLM-L6-v2", help="sentence-transformers model for real numbers.") ap.add_argument("--offline", action="store_true", @@ -141,9 +164,14 @@ def main() -> int: "recommended for turn-level dialogue datasets).") ap.add_argument("--json", dest="json_out", default=None, help="Also write the full JSON report to this path.") - args = ap.parse_args() + args = ap.parse_args(argv) + if args.canonical and args.limit is not None: + ap.error("--canonical rejects --limit; canonical artifacts must score every source case") cases = LOADERS[args.format](args.dataset, limit=args.limit) + if args.canonical and len(cases) != source_case_count(args.dataset): + print("canonical run rejected: normalization excluded source cases", file=sys.stderr) + return 2 if not cases: print("no usable cases found — is the file the right format?") return 2 @@ -166,12 +194,15 @@ def main() -> int: report["embedder"] = embedder_name report["measures"] = "retrieval (evidence recall@k), not end-to-end QA accuracy" report["wall_seconds"] = round(dt, 1) + report["canonical"] = bool(args.canonical) print(f"\nEngraphis × {args.format} — {report['questions']} questions @ k={args.k} " f"({dt:.1f}s)") print(f" evidence recall@k : {report['recall_at_k']:.3f}") print(f" evidence hit@k : {report['hit_at_k']:.3f}") print(f" answer_token_recall : {report['answer_token_recall']:.3f}") + print(f" retrieval scored : {report['scored_questions']}/{report['questions']} " + f"(exclusions={len(report['exclusions'])})") if args.json_out: slim = {k: v for k, v in report.items() if k != "detail"} Path(args.json_out).write_text(json.dumps(slim, indent=2), encoding="utf-8") diff --git a/eval/harness.py b/eval/harness.py index 84600bd4..a6a6f899 100644 --- a/eval/harness.py +++ b/eval/harness.py @@ -31,18 +31,262 @@ from __future__ import annotations import argparse +from dataclasses import asdict, dataclass import json from pathlib import Path -from typing import Optional +import subprocess +import time +from typing import Callable, Optional from engraphis.backends import DeterministicEmbedder, NumpyVectorIndex from engraphis.backends.reranker import IdentityReranker from engraphis.core.engine import MemoryEngine -from engraphis.core.interfaces import MemoryType, Scope +from engraphis.core.context import DeterministicContextPacker, RegexTokenCounter +from engraphis.core.grounded import build_grounded_answer +from engraphis.core.interfaces import ( + ContextUsage, Edge, MemoryRecord, MemoryType, Node, PackedChunk, Scope, SearchFilter, +) +from engraphis.core.recall import RecallResult +from engraphis.core.retrieval_policy import ProfileConfig from engraphis.core.store import Store +from eval.benchmark import ( + CANONICAL_TOKEN_BUDGETS, + canonical_benchmark_config, + exclusion, + paired_bootstrap_ci, + question_record, + report_envelope, + sha256_text, + stratified_bootstrap_ci, + validate_canonical_profile, + write_canonical_artifact, +) from eval import metrics +class _PinnedReaderTokenCounter: + """Count reader content tokens with a tokenizer loaded at one immutable revision.""" + + def __init__(self, tokenizer: object, identity: str) -> None: + self.tokenizer = tokenizer + self.identity = identity + + def __call__(self, text: str) -> int: + encode = getattr(self.tokenizer, "encode") + try: + return len(encode(text, add_special_tokens=False)) + except TypeError: + return len(encode(text)) + + +def _load_pinned_reader_token_counter(model: str, revision: str) -> Callable[[str], int]: + """Load the canonical reader tokenizer without affecting the offline default.""" + try: + from transformers import AutoProcessor + except ImportError as exc: # pragma: no cover - optional canonical benchmark dependency + raise ValueError( + "canonical output requires transformers and the pinned reader tokenizer" + ) from exc + processor = AutoProcessor.from_pretrained(model, revision=revision) + tokenizer = getattr(processor, "tokenizer", processor) + if not hasattr(tokenizer, "encode"): + raise ValueError("canonical reader processor did not expose an encode-capable tokenizer") + return _PinnedReaderTokenCounter(tokenizer, f"{model}@{revision}") + + +@dataclass(frozen=True) +class BaselineSpec: + """One baseline with an explicit executable mode and recorded limitations.""" + + label: str + retrieval_profile: str + vector: bool + lexical: bool + graph: bool + no_retrieval: bool = False + mode: str = "retrieval" + disable_temporal_resolution: bool = False + disable_reranker: bool = False + requires_nonidentity_reranker: bool = False + equivalent_to: Optional[str] = None + + @property + def arm_config(self) -> ProfileConfig: + return ProfileConfig( + self.label, vector=self.vector, lexical=self.lexical, + graph=self.graph, code=False, + ) + + def as_dict(self) -> dict: + return { + "label": self.label, + "retrieval_profile": self.retrieval_profile, + "arms": { + "vector": self.vector, + "lexical": self.lexical, + "graph": self.graph, + "code": False, + }, + "no_retrieval": self.no_retrieval, + "mode": self.mode, + "temporal_resolution": "disabled" if self.disable_temporal_resolution else "enabled", + "reranker": "disabled" if self.disable_reranker else "enabled", + **({"equivalent_to": self.equivalent_to} if self.equivalent_to else {}), + } + + +_EXECUTABLE_BASELINES = { + "full_hybrid": BaselineSpec("full_hybrid", "balanced", True, True, True), + "dense_only": BaselineSpec("dense_only", "balanced", True, False, False), + "lexical_only": BaselineSpec("lexical_only", "lexical", False, True, False), + # The current retrieval pipeline uses RRF whenever more than one arm is on. + # With graph disabled this is operationally identical to ``no_graph``; retain + # the published label, but make that equivalence visible in every artifact. + "dense_lexical_rrf": BaselineSpec( + "dense_lexical_rrf", "balanced", True, True, False, equivalent_to="no_graph", + ), + "full_history": BaselineSpec( + "full_history", "balanced", False, False, False, mode="full_history", + ), + "no_graph": BaselineSpec("no_graph", "balanced", True, True, False), + "no_reranker": BaselineSpec( + "no_reranker", "balanced", True, True, True, disable_reranker=True, + requires_nonidentity_reranker=True, + ), + "no_temporal_resolution": BaselineSpec( + "no_temporal_resolution", "balanced", True, True, True, + disable_temporal_resolution=True, + ), + "whole_document": BaselineSpec( + "whole_document", "balanced", False, False, False, mode="whole_document", + ), + "no_retrieval": BaselineSpec("no_retrieval", "balanced", False, False, False, True), +} + + +def executable_baseline(label: str) -> BaselineSpec: + """Return an honest harness baseline or fail before producing an artifact.""" + normalized = str(label or "").strip().casefold() + if normalized not in _EXECUTABLE_BASELINES: + supported = ", ".join(sorted(_EXECUTABLE_BASELINES)) + raise ValueError( + f"baseline_label {label!r} is not executable by eval.harness; " + f"supported labels: {supported}" + ) + return _EXECUTABLE_BASELINES[normalized] + + +def _validate_baseline_dataset(dataset: list[dict], baseline: BaselineSpec, reranker: object) -> None: + """Fail before an artifact when a claimed ablation has no representable input.""" + if baseline.mode == "whole_document" and not dataset: + raise ValueError("whole_document requires a non-empty dataset") + if baseline.mode == "whole_document" and not all( + isinstance(case.get("document"), str) and case["document"].strip() for case in dataset + ): + raise ValueError("whole_document requires a non-empty document in every dataset case") + if baseline.mode == "full_history" and not dataset: + raise ValueError("full_history requires a non-empty dataset") + if baseline.mode == "full_history" and not all( + isinstance(case.get("memories"), list) and case["memories"] for case in dataset + ): + raise ValueError("full_history requires ordered non-empty memories in every dataset case") + if baseline.disable_temporal_resolution: + groups: list[list[dict]] = [] + for case in dataset: + grouped: dict[tuple[str, str], list[dict]] = {} + for item in case.get("memories", []): + key = (str(item.get("subject_key", "")).strip(), str(item.get("claim_kind", "")).strip()) + if key[0]: + grouped.setdefault(key, []).append(item) + groups.extend(grouped.values()) + representable = any( + len(group) >= 2 + and len({str(item.get("text", "")) for item in group}) >= 2 + and all(item.get("valid_from") is not None for item in group) + for group in groups + ) + if not representable: + raise ValueError( + "no_temporal_resolution requires two memories with the same non-empty " + "subject_key (and claim_kind)" + ) + if baseline.requires_nonidentity_reranker and isinstance(reranker, IdentityReranker): + raise ValueError("no_reranker requires a non-identity reranker to make the ablation meaningful") + + +def _whole_source_result( + records: list[MemoryRecord], + *, + label: str, + token_budget: Optional[int], + token_counter: Optional[Callable[[str], int]] = None, + token_counter_identity: Optional[str] = None, +) -> RecallResult: + """Return exact source text for corpus baselines, never query-selecting or truncating it.""" + counter = token_counter or RegexTokenCounter() + counter_identity = ( + token_counter_identity + or getattr(counter, "identity", None) + or type(counter).__name__ + ) + context = "\n\n".join(record.content for record in records) + tokens = counter(context) + if token_budget is not None and tokens > int(token_budget): + raise ValueError(f"{label} cannot preserve complete source under token_budget={token_budget}") + packed = [PackedChunk( + id=record.id, excerpt=record.content, tokens=counter(record.content), reason=label, + ) for record in records] + usage = ContextUsage( + budget_tokens=tokens if token_budget is None else int(token_budget), + context_tokens=tokens, source_tokens=tokens, saved_tokens=0, savings_ratio=0.0, + packed_count=len(packed), omitted_count=0, token_counter=counter_identity, + ) + return RecallResult( + chunks=[{"id": record.id, "title": record.title, "content": record.content} + for record in records], + context=context, count=len(records), packed_chunks=packed, usage=usage, + retrieval_profile=label, + token_counter=counter, + ) + + +def _recall_for_baseline( + engine: MemoryEngine, + query: str, + *, + workspace_id: str, + repo_id: str, + k: int, + token_budget: Optional[int], + baseline: BaselineSpec, + source_records: Optional[list[MemoryRecord]] = None, +) -> RecallResult: + """Run the declared arms directly, without expanding ``RetrievalPolicy``.""" + budget = engine.recall_engine.token_budget if token_budget is None else max(0, int(token_budget)) + flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id, include_ancestors=True) + if baseline.mode in {"full_history", "whole_document"}: + packer = engine.recall_engine.context_packer + return _whole_source_result( + source_records or [], + label=baseline.label, + token_budget=token_budget, + token_counter=getattr(packer, "count_tokens", None), + token_counter_identity=getattr(packer, "token_counter_identity", None), + ) + if baseline.no_retrieval: + context, packed, usage = engine.recall_engine.context_packer.pack(query, [], budget) + return RecallResult( + context=context, packed_chunks=packed, usage=usage, + retrieval_profile=baseline.label, + token_counter=getattr(engine.recall_engine.context_packer, "count_tokens", None), + ) + return engine.recall_engine.recall( + query, flt, k=k, token_budget=token_budget, + retrieval_profile=baseline.retrieval_profile, + arm_config=baseline.arm_config, + ) + + def load_dataset(path: str) -> list[dict]: items = [] for line in Path(path).read_text(encoding="utf-8").splitlines(): @@ -52,57 +296,505 @@ def load_dataset(path: str) -> list[dict]: return items +def _git_commit() -> str: + """Return the checked-out commit when available, without making it a dependency.""" + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL + ).strip() or "unknown" + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def _seed_case_graph( + store: Store, + *, + workspace_id: str, + repo_id: str, + case: dict, +) -> None: + """Persist an optional fixture graph before its memories are written. + + The ordinary harness previously ignored ``entities``/``edges`` even though + graph fixtures declare them. Seeding first also lets the production write + path persist exact memory↔entity incidence, so the harness measures the + shipped sparse graph arm rather than accidentally falling back to dense + retrieval alone. + """ + entity_ids: dict[str, str] = {} + for entity in case.get("entities", []): + name = str(entity[0]) + entity_ids[name] = store.upsert_entity(Node( + id="", + name=name, + ntype=(str(entity[1]) if len(entity) > 1 else "concept"), + workspace_id=workspace_id, + repo_id=repo_id, + )) + for edge in case.get("edges", []): + source = entity_ids.get(str(edge[0])) + target = entity_ids.get(str(edge[1])) + if source is None or target is None: + raise ValueError( + f"eval edge references an unknown entity: {edge[0]!r} -> {edge[1]!r}" + ) + store.upsert_edge(Edge( + id="", + src=source, + dst=target, + relation=(str(edge[2]) if len(edge) > 2 else "rel"), + workspace_id=workspace_id, + repo_id=repo_id, + )) + + +def _usage_dict(usage, *, budget: int) -> dict: + """Keep the public v2 usage contract complete even for an empty recall.""" + if usage is not None: + return asdict(usage) + return { + "budget_tokens": budget, + "context_tokens": 0, + "source_tokens": 0, + "saved_tokens": 0, + "savings_ratio": 0.0, + "packed_count": 0, + "omitted_count": 0, + "token_counter": "unknown", + } + + +def _mean(records: list[dict], field: str) -> float: + return sum(float(item.get(field, 0.0)) for item in records) / max(len(records), 1) + + +def _v2_metrics(records: list[dict], *, bootstrap_iterations: int) -> dict: + """Aggregate conventional retrieval scores plus deterministic uncertainty.""" + scored = [item for item in records if not item.get("excluded")] + metric_fields = [ + "recall_at_1", "recall_at_5", "recall_at_10", + "mrr_at_1", "mrr_at_5", "mrr_at_10", + "ndcg_at_1", "ndcg_at_5", "ndcg_at_10", + ] + summary = {field: round(_mean(scored, field), 6) for field in metric_fields} + summary["answer_token_recall"] = round(_mean(scored, "answer_token_recall"), 6) + summary["confidence_intervals"] = { + field: stratified_bootstrap_ci( + scored, + lambda rows, metric=field: _mean(list(rows), metric), + iterations=bootstrap_iterations, + ) + for field in metric_fields + } + # A paired interval is meaningful only when a baseline contains the same + # question IDs. Keep the stable field present so artifact consumers never + # mistake an absent comparison for a zero-effect result. + summary["paired_bootstrap"] = { + "available": False, + "reason": "baseline_records_not_supplied", + "n": 0, + "delta": None, + "low": None, + "high": None, + "iterations": bootstrap_iterations, + } + labeled = [item for item in records if isinstance(item.get("answerable"), bool)] + grounded = [item for item in labeled if "grounded" in item and "abstained" in item] + if not labeled: + summary["grounded"] = { + "available": False, "reason": "no_answerability_labels", "n": 0, + } + summary["abstention"] = { + "available": False, "reason": "no_answerability_labels", "n": 0, + } + elif len(grounded) != len(labeled): + reason = "grounded_recall_not_run" + summary["grounded"] = {"available": False, "reason": reason, "n": len(labeled)} + summary["abstention"] = {"available": False, "reason": reason, "n": len(labeled)} + else: + answerable = [bool(item["answerable"]) for item in grounded] + summary["grounded"] = { + "available": True, + **metrics.grounded_precision_recall_f1( + [bool(item["grounded"]) for item in grounded], answerable, + ), + } + summary["abstention"] = { + "available": True, + **metrics.abstention_precision_recall_f1( + [bool(item["abstained"]) for item in grounded], answerable, + ), + } + for source, target in (("grounded", "grounded_f1"), ("abstention", "abstention_f1")): + measurement = summary[source] + summary[target] = ( + measurement["f1"] + if measurement["available"] + else {"available": False, "reason": measurement["reason"], "n": measurement["n"]} + ) + return summary + + +def paired_v2_bootstrap( + candidate_records: list[dict], baseline_records: list[dict], *, + metric: str = "recall_at_5", iterations: int = 1000, +) -> dict: + """Compute a paired interval after requiring complete question-ID coverage. + + A partial baseline would make an apparent delta incomparable, so it is + rejected rather than silently intersected away. This helper lets a caller + add a comparison after two independently written v2 runs. + """ + candidate = {item["question_id"]: item for item in candidate_records if not item.get("excluded")} + baseline = {item["question_id"]: item for item in baseline_records if not item.get("excluded")} + if set(candidate) != set(baseline): + raise ValueError("paired bootstrap requires identical scored question IDs") + result = paired_bootstrap_ci( + [(float(candidate[qid].get(metric, 0.0)), float(baseline[qid].get(metric, 0.0))) + for qid in sorted(candidate)], + iterations=iterations, + ) + return {"available": True, "metric": metric, **result} + + def run(dataset: list[dict], *, k: int = 5, dim: int = 256, embedder: Optional[DeterministicEmbedder] = None, - resolve_conflicts: bool = True) -> dict: + reranker: Optional[object] = None, grounded: bool = False, + resolve_conflicts: bool = True, v2: bool = False, + dataset_path: Optional[str] = None, token_budget: Optional[int] = None, + canonical: bool = False, canonical_profile: Optional[dict] = None, + bootstrap_iterations: int = 1000, + baseline_label: str = "full_hybrid") -> dict: + """Run the offline gate, or build the opt-in reproducible v2 envelope. + + The default output remains the original compact report. ``v2=True`` is + deliberately explicit because artifacts carry per-question measurements and + immutable provenance rather than only the CI gate's aggregate fields. + """ + if canonical and not v2: + v2 = True + if v2 and not dataset_path: + raise ValueError("v2 output requires dataset_path so the dataset can be hashed") + baseline = executable_baseline(baseline_label) + configured_reranker = reranker or IdentityReranker() + _validate_baseline_dataset(dataset, baseline, configured_reranker) + if canonical: + profile_errors = validate_canonical_profile(canonical_profile) + if profile_errors: + raise ValueError("canonical output requires pinned revisions: " + "; ".join(profile_errors)) + if canonical_profile["baseline_label"] != baseline.label: + raise ValueError( + "canonical_profile.baseline_label must match the executed baseline_label " + f"({baseline.label})" + ) + if not dataset: + raise ValueError("canonical output requires a complete, non-empty dataset") + if ( + not isinstance(bootstrap_iterations, int) + or isinstance(bootstrap_iterations, bool) + or bootstrap_iterations <= 0 + ): + raise ValueError( + "canonical output requires a positive bootstrap_iterations value" + ) + reader_profile = canonical_profile["reader"] + context_token_counter = _load_pinned_reader_token_counter( + reader_profile["model"], reader_profile["revision"] + ) + context_token_method = "pinned_reader_content_tokenizer" + context_tokenizer_identity = ( + f"{reader_profile['model']}@{reader_profile['revision']}" + ) + else: + context_token_counter = None + context_token_method = "deterministic_estimate" + context_tokenizer_identity = None embedder = embedder or DeterministicEmbedder(dim=dim) per_q = [] + curve_measurements = {budget: [] for budget in CANONICAL_TOKEN_BUDGETS} if canonical else {} for case in dataset: store = Store(":memory:") wid = store.get_or_create_workspace("eval") rid = store.get_or_create_repo(wid, case.get("id", "case")) index = NumpyVectorIndex(store) - engine = MemoryEngine(store, embedder, index, IdentityReranker()) + engine = MemoryEngine( + store, embedder, index, + None if baseline.disable_reranker else configured_reranker, + ) + if context_token_counter is not None: + engine.recall_engine.context_packer = DeterministicContextPacker( + token_counter=context_token_counter, + token_counter_identity=context_tokenizer_identity, + ) + _seed_case_graph( + store, + workspace_id=wid, + repo_id=rid, + case=case, + ) tag_to_id: dict[str, str] = {} id_to_tags: dict[str, list[str]] = {} id_to_text: dict[str, str] = {} - for m in case["memories"]: + document_record: Optional[MemoryRecord] = None + if baseline.mode == "whole_document": + document = str(case["document"]) mid = engine.remember( - m["text"], workspace_id=wid, repo_id=rid, mtype=MemoryType.EPISODIC, - scope=Scope.REPO, resolve_conflicts=resolve_conflicts, + document, workspace_id=wid, repo_id=rid, mtype=MemoryType.EPISODIC, + scope=Scope.REPO, title=str(case.get("id", "whole_document")), + resolve_conflicts=False, ) - tag = m.get("tag") - tag_to_id[tag] = mid - id_to_tags.setdefault(mid, []).append(tag) - id_to_text[mid] = m["text"] + document_record = store.get_memory(mid) + if document_record is None: # pragma: no cover - Store contract + raise RuntimeError("whole_document ingestion did not create a memory") + tag_to_id["whole_document"] = mid + # A whole-document baseline injects the complete case without query + # selection. It therefore contains every source tag in the case, + # not merely a synthetic document label. Otherwise a fixture that + # retains normal gold source IDs beside ``document`` would be + # incorrectly reported as a retrieval failure. + source_tags = [ + str(memory.get("tag")) + for memory in case.get("memories", []) + if memory.get("tag") is not None + ] + id_to_tags[mid] = source_tags or ["whole_document"] + id_to_text[mid] = document + else: + for m in case["memories"]: + mid = engine.remember( + m["text"], workspace_id=wid, repo_id=rid, mtype=MemoryType.EPISODIC, + scope=Scope.REPO, title=str(m.get("title", "")), + valid_from=m.get("valid_from"), subject_key=str(m.get("subject_key", "")), + claim_kind=str(m.get("claim_kind", "")), + resolve_conflicts=(False if baseline.disable_temporal_resolution else resolve_conflicts), + ) + tag = m.get("tag") + tag_to_id[tag] = mid + id_to_tags.setdefault(mid, []).append(tag) + id_to_text[mid] = m["text"] - for q in case["questions"]: - res = engine.recall(q["q"], workspace_id=wid, k=k) + history_records = ( + store.list_memories( + SearchFilter(workspace_id=wid, repo_id=rid, include_ancestors=True), + include_invalid=True, + ) if baseline.mode == "full_history" else None + ) + if history_records is not None: + history_records.sort(key=lambda record: (record.valid_from or record.ingested_at or 0.0, record.id)) + + for question_number, q in enumerate(case["questions"]): + question_id = str(q.get("id") or f"{case.get('id')}:{question_number}") + started = time.perf_counter_ns() + res = _recall_for_baseline( + engine, q["q"], workspace_id=wid, repo_id=rid, k=k, + token_budget=token_budget, baseline=baseline, + source_records=(history_records if history_records is not None else + ([document_record] if document_record is not None else None)), + ) + latency_ms = (time.perf_counter_ns() - started) / 1_000_000 retrieved_ids = [c["id"] for c in res.chunks] retrieved_tags = [t for i in retrieved_ids for t in id_to_tags.get(i, [None])] retrieved_texts = [id_to_text.get(i, "") for i in retrieved_ids] - supporting = q.get("supporting", []) - per_q.append({ - "case": case.get("id"), - "q": q["q"], - "recall_at_k": metrics.recall_at_k(retrieved_tags, supporting), - "hit_at_k": metrics.hit_at_k(retrieved_tags, supporting), - "answer_token_recall": metrics.answer_token_recall(retrieved_texts, q.get("answer", "")), - }) + supporting = q.get("supporting", ["whole_document"] if document_record else []) + excluded = None + if q.get("answerable") is False: + excluded = exclusion( + str(q.get("id") or f"{case.get('id')}:{question_number}"), + str(q.get("exclusion_reason") or "no_gold_evidence"), + ) + depth_metrics = metrics.retrieval_metrics_at_depths( + retrieved_tags, supporting, depths=(1, 5, 10), + ) + answerable = q.get("answerable") + grounded_answer = ( + build_grounded_answer(q["q"], res, engine.embedder) + if grounded and isinstance(answerable, bool) else None + ) + usage = _usage_dict( + res.usage, + budget=(token_budget if token_budget is not None else 1500), + ) + record = question_record( + question_id, + category=str(q.get("category") or "unknown"), + retrieved_ids=[tag for tag in retrieved_tags if tag], + supporting_ids=supporting, + context_tokens=usage["context_tokens"], + latency_ms=latency_ms, + excluded=excluded, + case=case.get("id"), q=q["q"], + **({"answerable": answerable} if isinstance(answerable, bool) else {}), + **({"grounded": grounded_answer.grounded, + "abstained": grounded_answer.abstained, + "grounded_support": round(grounded_answer.support, 6)} + if grounded_answer is not None else {}), + recall_at_k=metrics.recall_at_k(retrieved_tags, supporting), + hit_at_k=metrics.hit_at_k(retrieved_tags, supporting), + mrr_at_k=metrics.mrr_at_k(retrieved_tags, supporting, k), + ndcg_at_k=metrics.ndcg_at_k(retrieved_tags, supporting, k), + answer_token_recall=metrics.answer_token_recall( + retrieved_texts, q.get("answer", q.get("evidence", "")), + ), + usage=usage, + **depth_metrics, + ) + # Legacy terminal reports retain their query text for local + # debugging. Public v2 artifacts receive only this irreversible + # fingerprint, preventing a report from exporting a private prompt + # merely because it records per-question measurements. + record["question_sha256"] = sha256_text(str(q["q"])) + record["context_token_method"] = context_token_method + if context_tokenizer_identity is not None: + record["context_tokenizer_identity"] = context_tokenizer_identity + per_q.append(record) + if canonical: + for budget in CANONICAL_TOKEN_BUDGETS: + budget_result = _recall_for_baseline( + engine, q["q"], workspace_id=wid, repo_id=rid, k=k, + token_budget=budget, baseline=baseline, + source_records=(history_records if history_records is not None else + ([document_record] if document_record is not None else None)), + ) + # Fixed-budget quality is defined by evidence actually admitted + # to the packed context. Scoring the uncapped retrieval list + # would credit gold memories that the reader never received. + budget_ids = [chunk.id for chunk in budget_result.packed_chunks] + budget_tags = [ + tag for memory_id in budget_ids for tag in id_to_tags.get(memory_id, [None]) + ] + budget_depth = metrics.retrieval_metrics_at_depths( + budget_tags, supporting, depths=(1, 5, 10), + ) + budget_usage = _usage_dict(budget_result.usage, budget=budget) + curve_measurements[budget].append({ + "question_id": question_id, + "excluded": bool(excluded), + "context_tokens": budget_usage["context_tokens"], + "context_token_method": context_token_method, + "context_tokenizer_identity": context_tokenizer_identity, + "retrieved_ids": [tag for tag in budget_tags if tag], + "supporting_ids": list(supporting), + **budget_depth, + }) store.close() - n = max(len(per_q), 1) + scored = [item for item in per_q if not item.get("excluded")] + n = max(len(scored), 1) report = { "questions": len(per_q), - "recall_at_k": round(sum(x["recall_at_k"] for x in per_q) / n, 4), - "hit_at_k": round(sum(x["hit_at_k"] for x in per_q) / n, 4), - "answer_token_recall": round(sum(x["answer_token_recall"] for x in per_q) / n, 4), + "scored_questions": len(scored), + "exclusions": [item["excluded"] for item in per_q if item.get("excluded")], + "recall_at_k": round(sum(x["recall_at_k"] for x in scored) / n, 4), + "hit_at_k": round(sum(x["hit_at_k"] for x in scored) / n, 4), + "mrr_at_k": round(sum(x["mrr_at_k"] for x in scored) / n, 4), + "ndcg_at_k": round(sum(x["ndcg_at_k"] for x in scored) / n, 4), + "answer_token_recall": round(sum(x["answer_token_recall"] for x in scored) / n, 4), "k": k, + "baseline_label": baseline.label, + "baseline_execution": baseline.as_dict(), + "grounded_recall": bool(grounded), "detail": per_q, } - return report + if not v2: + return report + + profile = canonical_profile if canonical else None + config = { + "k": int(k), + "dim": int(dim), + "token_budget": token_budget, + "resolve_conflicts": bool(resolve_conflicts), + "grounded_recall": bool(grounded), + "bootstrap_iterations": int(bootstrap_iterations), + "baseline_label": baseline.label, + "baseline_execution": baseline.as_dict(), + } + if canonical: + config.update(canonical_benchmark_config( + run_label="eval.harness", baseline_label=baseline.label, + token_budgets=CANONICAL_TOKEN_BUDGETS, profile=profile, + )) + public_records = [] + for record in per_q: + public_record = dict(record) + public_record.pop("q", None) + public_records.append(public_record) + v2_metrics = _v2_metrics(per_q, bootstrap_iterations=max(0, int(bootstrap_iterations))) + if canonical: + v2_metrics["fixed_budget_curve"] = _measured_fixed_budget_curve(curve_measurements) + envelope = report_envelope( + suite="engraphis-harness", + dataset_path=dataset_path, + config=config, + records=public_records, + metrics=v2_metrics, + exclusions=report["exclusions"], + git_commit=_git_commit(), + ) + model = { + "name": type(embedder).__name__, + "model_id": getattr(embedder, "model_name", None), + "revision": getattr(embedder, "revision", None), + "dimension": getattr(embedder, "dim", dim), + } + envelope["models"] = {"embedder": {**model, "sha256": sha256_text(json.dumps(model, sort_keys=True))}} + envelope["legacy_summary"] = {key: value for key, value in report.items() if key != "detail"} + if canonical: + expected_embedding = profile["embedding"] + if model["model_id"] != expected_embedding["model"] or model["revision"] != expected_embedding["revision"]: + raise ValueError( + "canonical output requires an embedder whose model_name and revision match " + "canonical_profile.embedding" + ) + envelope["protocol"]["complete_dataset"] = True + envelope["protocol"]["source_questions"] = len(per_q) + return envelope + + +def _measured_fixed_budget_curve(measurements: dict[int, list[dict]]) -> dict: + """Summarize actual canonical budget reruns with their per-question evidence.""" + rows = [] + for budget in CANONICAL_TOKEN_BUDGETS: + records = sorted(measurements.get(budget, []), key=lambda item: item["question_id"]) + scored = [item for item in records if not item.get("excluded")] + row = { + "token_budget": budget, + "status": "measured", + "n_total": len(records), + "n_scored": len(scored), + "records": records, + } + row.update({field: round(_mean(scored, field), 6) for field in ( + "recall_at_1", "recall_at_5", "recall_at_10", + "mrr_at_1", "mrr_at_5", "mrr_at_10", + "ndcg_at_1", "ndcg_at_5", "ndcg_at_10", + )}) + rows.append(row) + return {"available": True, "rows": rows} + + +def run_baseline_matrix( + dataset: list[dict], + *, + baseline_labels: tuple[str, ...] = tuple(_EXECUTABLE_BASELINES), + **kwargs, +) -> dict[str, dict]: + """Run an explicit, reproducible matrix of executable harness baselines. + + A canonical artifact represents one declared method, so callers must run the + rows separately with their matching pinned profile rather than claiming a + multi-baseline canonical report. + """ + if kwargs.get("canonical"): + raise ValueError("run_baseline_matrix does not emit multi-baseline canonical artifacts") + if "baseline_label" in kwargs: + raise ValueError("pass labels through baseline_labels, not baseline_label") + return { + label: run(dataset, baseline_label=label, **kwargs) + for label in baseline_labels + } def _print(report: dict) -> None: @@ -112,16 +804,54 @@ def _print(report: dict) -> None: print(f" answer_token_recall : {report['answer_token_recall']:.3f}\n") -def main() -> None: +def main(argv: Optional[list[str]] = None) -> None: ap = argparse.ArgumentParser(description="Run the Engraphis retrieval eval.") ap.add_argument("--dataset", default=str(Path(__file__).resolve().parent / "datasets" / "sample.jsonl")) ap.add_argument("--k", type=int, default=5) ap.add_argument("--dim", type=int, default=256) + ap.add_argument("--token-budget", type=int, default=None, + help="packed-context token budget (recorded in v2 artifacts)") ap.add_argument("--json", action="store_true", help="print full JSON report") - args = ap.parse_args() + ap.add_argument("--v2", action="store_true", + help="emit the provenance-complete engraphis-benchmark/v2 envelope") + ap.add_argument("--artifact", default=None, + help="immutably write a v2 JSON artifact and SHA-256 sidecar") + ap.add_argument("--canonical", action="store_true", + help="require a complete dataset and a pinned canonical profile (implies --v2)") + ap.add_argument("--canonical-profile", default=None, + help="JSON file with pinned benchmark, reader, and embedding revisions") + ap.add_argument("--baseline-label", default="full_hybrid", + help="executable baseline: " + ", ".join(sorted(_EXECUTABLE_BASELINES))) + ap.add_argument("--bootstrap-iterations", type=int, default=1000, + help="deterministic stratified-bootstrap iterations for v2 output") + ap.add_argument("--grounded", action="store_true", + help="run deterministic grounded recall for rows declaring answerable") + args = ap.parse_args(argv) + + if args.artifact and not (args.v2 or args.canonical): + ap.error("--artifact requires --v2 (or --canonical)") + if args.canonical and not args.canonical_profile: + ap.error("--canonical requires --canonical-profile with pinned revisions") + profile = None + if args.canonical_profile: + try: + profile = json.loads(Path(args.canonical_profile).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + ap.error(f"could not read --canonical-profile: {exc}") - report = run(load_dataset(args.dataset), k=args.k, dim=args.dim) - if args.json: + try: + report = run( + load_dataset(args.dataset), k=args.k, dim=args.dim, + v2=args.v2 or args.canonical, dataset_path=args.dataset, + token_budget=args.token_budget, canonical=args.canonical, + canonical_profile=profile, bootstrap_iterations=args.bootstrap_iterations, + baseline_label=args.baseline_label, grounded=args.grounded, + ) + if args.artifact: + write_canonical_artifact(report, args.artifact, canonical=args.canonical) + except ValueError as exc: + ap.error(str(exc)) + if args.json or args.v2 or args.canonical: print(json.dumps(report, indent=2)) else: _print(report) diff --git a/eval/longmemeval_v2.py b/eval/longmemeval_v2.py new file mode 100644 index 00000000..58bd7ce3 --- /dev/null +++ b/eval/longmemeval_v2.py @@ -0,0 +1,615 @@ +"""Adapter for LongMemEval-V2's official ``Memory.insert/query`` contract. + +The official harness owns downloading data, model calls, and scoring. This +adapter deliberately does none of those things: it exposes an Engraphis-backed +memory object with the same two public methods for use inside that harness. +""" +from __future__ import annotations + +from collections.abc import Callable +import json +from pathlib import Path +import re +import sqlite3 +import threading +from typing import Any, Optional, Protocol, Sequence, Union + +from engraphis.core.context import RegexTokenCounter +from engraphis.service import MemoryService +from eval.benchmark import CANONICAL_TOKEN_BUDGETS + + +_PINNED_REVISION = re.compile(r"[0-9a-f]{40}\Z") +_MAX_TRAJECTORY_CHUNK_CHARS = 1800 + + +def _require_configured_embedder(service: MemoryService, model: Optional[str], + revision: Optional[str]) -> None: + """Fail closed when a benchmark-requested embedder silently fell back. + + The general Engraphis factory intentionally degrades to its offline deterministic + embedder when an optional model cannot load. That is useful product behavior but + invalid for a canonical benchmark: the resulting artifact would otherwise name a + Qwen revision that never produced its vectors. + """ + if not model: + return + engine = getattr(service, "engine", None) + embedder = getattr(engine, "embedder", None) + if ( + getattr(embedder, "model_name", None) != model + or getattr(embedder, "revision", None) != revision + ): + raise RuntimeError( + "the configured benchmark embedder did not load at its pinned revision; " + "canonical fallback is forbidden" + ) + + +# LongMemEval-V2 commit 6f020ac2fc3275e46c706d3406e02c3ed79b7be2 +# exposes this interface as ``memory_modules.memory``. It is an optional +# benchmark dependency: importing this module must continue to work in the +# normal offline Engraphis test environment. +try: # pragma: no cover - exercised in an isolated fake-official-package test + from memory_modules.memory import Memory as _MemoryBase + from memory_modules.memory import register_memory as _register_memory + OFFICIAL_MEMORY_AVAILABLE = True +except Exception: # noqa: BLE001 - third-party optional import boundary + OFFICIAL_MEMORY_AVAILABLE = False + + class _MemoryBase: + """Small compatible fallback for local/offline use only.""" + + memory_type = "" + + def __init__(self, memory_params: dict[str, object]) -> None: + self.memory_params = dict(memory_params) + + @property + def memory_config(self) -> dict[str, object]: + return {"memory_type": self.memory_type, "memory_params": self.memory_params} + + def configure_runtime(self, **kwargs: object) -> None: + del kwargs + + def save_memory(self, output_dir: str | Path) -> None: + path = Path(output_dir) + path.mkdir(parents=True, exist_ok=True) + (path / "memory_config.json").write_text( + json.dumps(self.memory_config, indent=2, ensure_ascii=True) + "\n", + encoding="utf-8", + ) + self._save_backend(path) + + def _save_backend(self, output_dir: Path) -> None: + del output_dir + + def _load_backend(self, input_dir: Path) -> None: + del input_dir + + def _register_memory(memory_cls): + return memory_cls + + +class ContextTokenizer(Protocol): + """Small tokenizer surface accepted by the official-harness adapter.""" + + def encode(self, text: str) -> Sequence[Any]: + ... + + +def _reader_tokenizer_identity(model: str, revision: str) -> str: + """Return the immutable identity recorded for canonical reader accounting.""" + return f"{model}@{revision}" + + +def _load_pinned_reader_tokenizer(model: str, revision: str) -> ContextTokenizer: + """Load the official reader tokenizer only for an explicitly canonical run. + + Importing transformers stays optional for the offline core. Passing the + immutable revision to ``from_pretrained`` is important: a mutable model tag + would make a token budget unverifiable even if its displayed model name was + unchanged. + """ + try: + # The official V2 harness builds prompts with ``AutoProcessor`` rather + # than loading a tokenizer directly. Use that exact public surface at + # the pinned revision and take its text tokenizer for preflight context + # accounting. This keeps the adapter's hard budget aligned with the + # reader path instead of merely using a similarly named tokenizer. + from transformers import AutoProcessor + except ImportError as exc: # pragma: no cover - depends on optional benchmark install + raise ValueError( + "canonical LongMemEval-V2 accounting requires transformers and the pinned " + "official reader processor/tokenizer" + ) from exc + processor = AutoProcessor.from_pretrained(model, revision=revision) + tokenizer = getattr(processor, "tokenizer", processor) + if not hasattr(tokenizer, "encode"): + raise ValueError( + "canonical LongMemEval-V2 reader processor did not expose an encode-capable tokenizer" + ) + return tokenizer + + +@_register_memory +class EngraphisLongMemEvalV2Memory(_MemoryBase): + """Minimal official-harness-compatible memory backend. + + ``insert`` accepts a full trajectory object. ``query`` returns the official + list of ``{"type": "text", "value": ...}`` context items. Image handling + remains the official harness's concern; trajectory text is indexed locally. + """ + + memory_type = "engraphis" + + def __init__( + self, + memory_params: Optional[dict[str, object]] = None, + *, + context_k: Optional[int] = None, + max_context_tokens: Optional[int] = None, + tokenizer: Optional[Union[ContextTokenizer, Callable[[str], int]]] = None, + tokenizer_identity: Optional[str] = None, + require_exact_reader_tokenizer: Optional[bool] = None, + reader_tokenizer_model: Optional[str] = None, + reader_tokenizer_revision: Optional[str] = None, + retrieval_profile: Optional[str] = None, + embed_model: Optional[str] = None, + embed_revision: Optional[str] = None, + vector_backend: Optional[str] = None, + service: Optional[MemoryService] = None, + ) -> None: + """Construct directly or from official ``memory_params`` configuration. + + The official ``build_memory`` calls ``memory_cls(memory_params)``. The + keyword form is retained for local tests and programmatic integrations. + Runtime-only dependencies such as a tokenizer or an already-open service + deliberately do not enter the persisted config. + """ + params = dict(memory_params or {}) + unknown = set(params) - { + "context_k", "max_context_tokens", "tokenizer_identity", "retrieval_profile", + "embed_model", "embed_revision", "vector_backend", "require_exact_reader_tokenizer", + "reader_tokenizer_model", "reader_tokenizer_revision", + } + if unknown: + raise ValueError("unsupported Engraphis LongMemEval memory_params: " + ", ".join(sorted(unknown))) + resolved_context_k = context_k if context_k is not None else params.get("context_k", 8) + resolved_max_tokens = ( + max_context_tokens if max_context_tokens is not None + else params.get("max_context_tokens", CANONICAL_TOKEN_BUDGETS[2]) + ) + resolved_profile = ( + retrieval_profile if retrieval_profile is not None + else params.get("retrieval_profile", "balanced") + ) + self.context_k = max(1, int(resolved_context_k)) + self.max_context_tokens = int(resolved_max_tokens) + if self.max_context_tokens <= 0: + raise ValueError("max_context_tokens must be positive") + self.require_exact_reader_tokenizer = bool( + require_exact_reader_tokenizer + if require_exact_reader_tokenizer is not None + else params.get("require_exact_reader_tokenizer", False) + ) + self.reader_tokenizer_model = str( + reader_tokenizer_model + if reader_tokenizer_model is not None + else params.get("reader_tokenizer_model") or "" + ).strip() or None + self.reader_tokenizer_revision = str( + reader_tokenizer_revision + if reader_tokenizer_revision is not None + else params.get("reader_tokenizer_revision") or "" + ).strip() or None + if self.require_exact_reader_tokenizer: + if not self.reader_tokenizer_model or not self.reader_tokenizer_revision: + raise ValueError( + "canonical LongMemEval-V2 accounting requires reader_tokenizer_model " + "and reader_tokenizer_revision" + ) + if _PINNED_REVISION.fullmatch(self.reader_tokenizer_revision) is None: + raise ValueError( + "reader_tokenizer_revision must be an immutable lowercase " + "40-character commit" + ) + expected_tokenizer_identity = _reader_tokenizer_identity( + self.reader_tokenizer_model, self.reader_tokenizer_revision + ) + declared_identity = tokenizer_identity or params.get("tokenizer_identity") + if declared_identity != expected_tokenizer_identity: + raise ValueError( + "canonical LongMemEval-V2 accounting requires the pinned reader " + "tokenizer identity" + ) + if tokenizer is not None: + raise ValueError( + "canonical LongMemEval-V2 accounting loads the pinned reader tokenizer " + "internally and does not accept an injected replacement" + ) + tokenizer = _load_pinned_reader_tokenizer( + self.reader_tokenizer_model, self.reader_tokenizer_revision + ) + tokenizer_identity = expected_tokenizer_identity + self._tokenizer = tokenizer or RegexTokenCounter() + self.tokenizer_identity = ( + tokenizer_identity + or params.get("tokenizer_identity") + or getattr(self._tokenizer, "identity", None) + or getattr(self._tokenizer, "__name__", None) + or type(self._tokenizer).__name__ + ) + self.retrieval_profile = str(resolved_profile or "balanced").strip().casefold() + self.embed_model = str( + embed_model if embed_model is not None else params.get("embed_model") or "" + ).strip() or None + self.embed_revision = str( + embed_revision if embed_revision is not None else params.get("embed_revision") or "" + ).strip() or None + self.vector_backend = str( + vector_backend if vector_backend is not None else params.get("vector_backend") or "numpy" + ).strip().casefold() + if self.embed_model and ( + self.embed_revision is None or _PINNED_REVISION.fullmatch(self.embed_revision) is None + ): + raise ValueError( + "embed_revision must be an immutable lowercase 40-character commit " + "when embed_model is configured" + ) + if self.embed_revision and not self.embed_model: + raise ValueError("embed_model is required when embed_revision is configured") + if self.vector_backend not in {"numpy", "sqlite-vec"}: + raise ValueError("vector_backend must be numpy or sqlite-vec") + persisted_params = { + "context_k": self.context_k, + "max_context_tokens": self.max_context_tokens, + "tokenizer_identity": self.tokenizer_identity, + "require_exact_reader_tokenizer": self.require_exact_reader_tokenizer, + "reader_tokenizer_model": self.reader_tokenizer_model, + "reader_tokenizer_revision": self.reader_tokenizer_revision, + "retrieval_profile": self.retrieval_profile, + "embed_model": self.embed_model, + "embed_revision": self.embed_revision, + "vector_backend": self.vector_backend, + } + super().__init__(persisted_params) + self.service = service or MemoryService.create( + ":memory:", + embed_model=self.embed_model, + embed_revision=self.embed_revision, + vector_backend=self.vector_backend, + ) + _require_configured_embedder( + self.service, self.embed_model, self.embed_revision + ) + self.workspace = "longmemeval-v2" + self.repo = "trajectory" + self._counter = 0 + self._query_result_local = threading.local() + + def configure_runtime(self, **kwargs: object) -> None: + """Accept an official-harness runtime tokenizer without changing config.""" + tokenizer = kwargs.pop("tokenizer", None) + tokenizer_identity = kwargs.pop("tokenizer_identity", None) + if self.require_exact_reader_tokenizer and ( + tokenizer is not None or tokenizer_identity is not None + ): + raise ValueError( + "canonical LongMemEval-V2 accounting does not permit runtime tokenizer replacement" + ) + if tokenizer is not None: + if not callable(tokenizer) and not hasattr(tokenizer, "encode"): + raise TypeError("tokenizer must be callable or expose encode()") + self._tokenizer = tokenizer # type: ignore[assignment] + if tokenizer_identity is not None: + self.tokenizer_identity = str(tokenizer_identity) + elif tokenizer is not None: + self.tokenizer_identity = ( + getattr(tokenizer, "identity", None) + or getattr(tokenizer, "__name__", None) + or type(tokenizer).__name__ + ) + super().configure_runtime(**kwargs) + + @property + def metadata(self) -> dict[str, Any]: + """Stable context-budget metadata for the official benchmark artifact.""" + return { + "memory_type": self.memory_type, + "context_k": self.context_k, + "max_context_tokens": self.max_context_tokens, + "budget_curve_status": "single_operating_point", + "required_budget_matrix": list(CANONICAL_TOKEN_BUDGETS), + "tokenizer": self.tokenizer_identity, + "token_budget_method": ( + "pinned_reader_content_tokenizer" + if self.require_exact_reader_tokenizer else "deterministic_estimate" + ), + "token_budget_scope": "per_context_item_content_excluding_prompt_framing", + "retrieval_profile": self.retrieval_profile, + "embed_model": self.embed_model or "deterministic", + "embed_revision": self.embed_revision, + "vector_backend": self.vector_backend, + "response_mode": "compact", + } + + def insert(self, trajectory: dict[str, Any]) -> None: + """Store one official trajectory without assuming its private schema.""" + trajectory_id = str(trajectory.get("trajectory_id") or trajectory.get("id") or self._counter) + segments = _trajectory_segments(trajectory) + if not segments: + return + sequence = 0 + for state_index, text in segments: + for chunk_index, chunk in enumerate(_split_trajectory_text(text), start=1): + sequence += 1 + self.service.remember( + chunk, + workspace=self.workspace, + repo=self.repo, + mtype="episodic", + scope="repo", + title=f"trajectory:{trajectory_id}:state:{state_index}:part:{chunk_index}", + metadata={ + "benchmark": "LongMemEval-V2", + "trajectory_id": trajectory_id, + "state_index": state_index, + "chunk_index": chunk_index, + "sequence": sequence, + }, + source="benchmark", + kind="longmemeval_v2", + resolve_conflicts=False, + ) + self._counter += 1 + + def query(self, query: str, query_image: Optional[str] = None) -> list[dict]: + """Return text context items accepted by LongMemEval-V2's reader harness.""" + del query_image # The official protocol permits it; text-only retrieval is explicit. + response = self.service.recall( + query, + workspace=self.workspace, + repo=self.repo, + k=self.context_k, + token_budget=self.max_context_tokens, + retrieval_profile=self.retrieval_profile, + response_mode="compact", + reinforce=False, + # The official harness can build prompts concurrently. Benchmark + # retrieval is observational: receipt writes would add contention, + # alter the SQLite database, and make repeated reader calls + # non-deterministic without contributing benchmark evidence. + record_receipt=False, + ) + # The packed context is compact by construction. Recount it under the + # pinned reader tokenizer because the engine's local counter may be + # different; clip once more to make the evidence-item content budget + # exact. Prompt framing and inter-item separators remain the official + # harness's responsibility and are deliberately not counted here. + items = _context_items_with_budget( + str(response.get("context") or ""), + budget=self.max_context_tokens, + count=self._count_tokens, + ) + self._query_result_local.metadata = { + "memory_type": self.memory_type, + "retrieval_profile": response.get("retrieval_profile"), + "source_ids": [ + source.get("id") + for source in response.get("packed_sources", []) + if source.get("id") + ], + "usage": response.get("usage", {}), + "returned_context_tokens": sum( + self._count_tokens(item["value"]) for item in items + ), + "returned_context_items": len(items), + "tokenizer": self.tokenizer_identity, + "token_budget_method": ( + "pinned_reader_content_tokenizer" + if self.require_exact_reader_tokenizer else "deterministic_estimate" + ), + "token_budget_scope": "per_context_item_content_excluding_prompt_framing", + "budget_curve_status": "single_operating_point", + } + return items + + def post_query_hook( + self, + *, + query: str, + query_image: Optional[str], + memory_context: list[dict], + ) -> dict[str, object]: + """Expose content-free Engraphis retrieval evidence to official run logs.""" + del query, query_image, memory_context + metadata = getattr(self._query_result_local, "metadata", None) + return dict(metadata) if isinstance(metadata, dict) else {} + + def _count_tokens(self, text: str) -> int: + if callable(self._tokenizer) and not hasattr(self._tokenizer, "encode"): + return max(0, int(self._tokenizer(text))) + try: + return len(self._tokenizer.encode(text, add_special_tokens=False)) + except TypeError: + return len(self._tokenizer.encode(text)) + + def _save_backend(self, output_dir: Path) -> None: + """Persist the local SQLite store through the official memory hook.""" + database = output_dir / "engraphis.sqlite" + target = sqlite3.connect(str(database)) + try: + self.service.store.conn.backup(target) + finally: + target.close() + (output_dir / "engraphis_state.json").write_text( + json.dumps({"counter": self._counter}, sort_keys=True) + "\n", + encoding="utf-8", + ) + + def _load_backend(self, input_dir: Path) -> None: + """Restore a SQLite store saved by :meth:`_save_backend`.""" + database = input_dir / "engraphis.sqlite" + if not database.is_file(): + raise FileNotFoundError(f"missing saved Engraphis SQLite backend: {database}") + self.service.store.close() + self.service = MemoryService.create( + str(database), + embed_model=self.embed_model, + embed_revision=self.embed_revision, + vector_backend=self.vector_backend, + ) + _require_configured_embedder( + self.service, self.embed_model, self.embed_revision + ) + state = input_dir / "engraphis_state.json" + if state.is_file(): + payload = json.loads(state.read_text(encoding="utf-8")) + self._counter = int(payload.get("counter", 0)) + + +def _fit_to_budget(text: str, budget: int, count: Callable[[str], int]) -> str: + """Return a deterministic prefix whose injected-token count fits ``budget``.""" + text = text.strip() + if not text or count(text) <= budget: + return text + # Prefix token counts are monotonic for normal reader tokenizers. The final + # loop remains a correctness guard for unusual injected implementations. + low, high = 0, len(text) + while low < high: + middle = (low + high + 1) // 2 + if count(text[:middle]) <= budget: + low = middle + else: + high = middle - 1 + clipped = text[:low].rstrip() + while clipped and count(clipped) > budget: + clipped = clipped[:-1].rstrip() + return clipped + + +def _context_items_with_budget( + context: str, + *, + budget: int, + count: Callable[[str], int], +) -> list[dict[str, str]]: + """Keep packed sources as separate official items under one aggregate budget. + + LongMemEval-V2 truncates context at item boundaries using the canonical + Qwen reader processor. Returning one monolithic item would therefore turn a + small tokenizer mismatch into *zero* evidence. Separate packed sources let + the official harness retain the largest exact-token prefix while this + adapter still enforces its configured counter exactly. + """ + blocks = [ + block.strip() + for block in re.split(r"\n{2,}(?=\[\d+\](?:\s|$))", context.strip()) + if block.strip() + ] + items: list[dict[str, str]] = [] + used = 0 + for block in blocks: + remaining = budget - used + if remaining <= 0: + break + fitted = _fit_to_budget(block, remaining, count) + if not fitted: + continue + items.append({"type": "text", "value": fitted}) + used += count(fitted) + return items + + +def _trajectory_text(trajectory: dict[str, Any]) -> str: + """Flatten current official and legacy trajectory shapes without model calls.""" + return "\n".join(text for _index, text in _trajectory_segments(trajectory)) + + +def _trajectory_segments(trajectory: dict[str, Any]) -> list[tuple[int, str]]: + """Return ordered state text without retaining a duplicate full trajectory. + + Official V2 trajectories have a ``states`` list. Indexing that whole list + as one memory lets an embedder truncate late states, so each useful state is + kept independently and then deterministically chunked by :meth:`insert`. + Legacy list shapes follow the same rule. Direct text is used only when no + structured state is available, avoiding a summary/full-history duplicate. + """ + segments: list[tuple[int, str]] = [] + for key in ("states", "content", "steps", "trajectory"): + items = trajectory.get(key) + if not isinstance(items, list): + continue + for state_index, item in enumerate(items): + lines: list[str] = [] + _append_trajectory_item(lines, item) + text = "\n".join(lines).strip() + if text: + segments.append((state_index, text)) + if segments: + return segments + for key in ("text", "trajectory_text", "notes"): + value = trajectory.get(key) + if isinstance(value, str) and value.strip(): + return [(0, value.strip())] + content = trajectory.get("content") + if isinstance(content, str) and content.strip(): + return [(0, content.strip())] + return [] + + +def _split_trajectory_text(text: str, *, max_chars: int = _MAX_TRAJECTORY_CHUNK_CHARS) -> list[str]: + """Split long state text deterministically without dropping or repeating text.""" + if max_chars <= 0: + raise ValueError("max_chars must be positive") + lines = [line.strip() for line in text.splitlines() if line.strip()] + if not lines: + return [] + chunks: list[str] = [] + current = "" + for line in lines: + # A single accessibility-tree line can itself exceed an embedder's + # useful input. Preserve every character in stable fixed-size pieces. + parts = [line[index:index + max_chars] for index in range(0, len(line), max_chars)] + for part in parts: + candidate = part if not current else current + "\n" + part + if current and len(candidate) > max_chars: + chunks.append(current) + current = part + else: + current = candidate + if current: + chunks.append(current) + return chunks + + +def _append_trajectory_item(lines: list[str], item: Any) -> None: + if isinstance(item, str): + if item.strip(): + lines.append(item.strip()) + return + if not isinstance(item, dict): + return + for key in ("accessibility_tree", "text", "action", "thought", "url"): + _append_trajectory_value(lines, key, item.get(key)) + _append_trajectory_value(lines, "thought", item.get("thoughts")) + observation = item.get("observation") + if isinstance(observation, dict): + for key in ("accessibility_tree", "text", "url"): + _append_trajectory_value(lines, f"observation.{key}", observation.get(key)) + else: + _append_trajectory_value(lines, "observation", observation) + content = item.get("content") + if isinstance(content, dict): + _append_trajectory_item(lines, content) + else: + _append_trajectory_value(lines, "content", content) + + +def _append_trajectory_value(lines: list[str], label: str, value: Any) -> None: + if isinstance(value, str) and value.strip(): + lines.append(f"{label}: {value.strip()}") + elif isinstance(value, list): + for part in value: + _append_trajectory_value(lines, label, part) diff --git a/eval/metrics.py b/eval/metrics.py index b5ee8955..de38602d 100644 --- a/eval/metrics.py +++ b/eval/metrics.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +import math + def recall_at_k(retrieved_ids: list[str], supporting_ids: list[str]) -> float: """Fraction of the gold supporting facts that appear in the retrieved set.""" @@ -19,6 +21,95 @@ def hit_at_k(retrieved_ids: list[str], supporting_ids: list[str]) -> float: return 1.0 if any(s in retrieved_ids for s in supporting_ids) else 0.0 +def reciprocal_rank(retrieved_ids: list[str], supporting_ids: list[str]) -> float: + """Reciprocal rank of the first evidence item (zero when none is retrieved).""" + supporting = set(supporting_ids) + for position, item in enumerate(retrieved_ids, start=1): + if item in supporting: + return 1.0 / position + return 0.0 + + +def mrr_at_k(retrieved_ids: list[str], supporting_ids: list[str], k: int) -> float: + """MRR truncated to a declared depth, suitable for per-question averaging.""" + return reciprocal_rank(retrieved_ids[:max(0, int(k))], supporting_ids) + + +def ndcg_at_k(retrieved_ids: list[str], supporting_ids: list[str], k: int) -> float: + """Binary-relevance normalized discounted cumulative gain at depth ``k``.""" + supporting = set(supporting_ids) + if not supporting: + return 1.0 + dcg = sum( + 1.0 / math.log2(position + 1) + for position, item in enumerate(retrieved_ids[:max(0, int(k))], start=1) + if item in supporting + ) + ideal = sum(1.0 / math.log2(position + 1) + for position in range(1, min(len(supporting), max(0, int(k))) + 1)) + return dcg / ideal if ideal else 0.0 + + +def retrieval_metrics_at_depths( + retrieved_ids: list[str], supporting_ids: list[str], depths: tuple[int, ...] = (1, 5, 10) +) -> dict: + """Return the conventional Recall/Hit/MRR/nDCG suite at declared depths.""" + result = {} + for depth in sorted(set(max(1, int(value)) for value in depths)): + retrieved = retrieved_ids[:depth] + result[f"recall_at_{depth}"] = recall_at_k(retrieved, supporting_ids) + result[f"hit_at_{depth}"] = hit_at_k(retrieved, supporting_ids) + result[f"mrr_at_{depth}"] = mrr_at_k(retrieved_ids, supporting_ids, depth) + result[f"ndcg_at_{depth}"] = ndcg_at_k(retrieved_ids, supporting_ids, depth) + return result + + +def binary_precision_recall_f1( + predicted_positive: list[bool], expected_positive: list[bool] +) -> dict[str, float | int]: + """Return transparent binary precision/recall/F1 with explicit counts. + + An empty predicted-positive set has precision 1 only when there are no true + positives either. This keeps an all-abstain system from receiving a free + precision score on answerable questions while making the all-negative edge + case well-defined. + """ + if len(predicted_positive) != len(expected_positive): + raise ValueError("predicted_positive and expected_positive must have equal length") + tp = sum(predicted and expected + for predicted, expected in zip(predicted_positive, expected_positive)) + fp = sum(predicted and not expected + for predicted, expected in zip(predicted_positive, expected_positive)) + fn = sum(not predicted and expected + for predicted, expected in zip(predicted_positive, expected_positive)) + precision = tp / (tp + fp) if tp + fp else (1.0 if not any(expected_positive) else 0.0) + recall = tp / (tp + fn) if tp + fn else 1.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + return { + "precision": precision, + "recall": recall, + "f1": f1, + "true_positive": tp, + "false_positive": fp, + "false_negative": fn, + "n": len(predicted_positive), + } + + +def grounded_precision_recall_f1( + grounded: list[bool], answerable: list[bool] +) -> dict[str, float | int]: + """Score grounded answers as positives against answerable questions.""" + return binary_precision_recall_f1(grounded, answerable) + + +def abstention_precision_recall_f1( + abstained: list[bool], answerable: list[bool] +) -> dict[str, float | int]: + """Score abstentions as positives against questions without answer evidence.""" + return binary_precision_recall_f1(abstained, [not item for item in answerable]) + + def answer_token_recall(retrieved_texts: list[str], answer: str) -> float: """Fraction of the gold answer's content tokens present in retrieved text.""" gold = _tokens(answer) diff --git a/eval/performance.py b/eval/performance.py new file mode 100644 index 00000000..981f1c8a --- /dev/null +++ b/eval/performance.py @@ -0,0 +1,764 @@ +"""Reproducible recall latency, quality, and context-efficiency benchmark. + +This measures the complete shipped recall path after ingestion: semantic + lexical + +graph candidate generation, fusion, scoring, reranking, context packing, and temporal +visibility. It deliberately disables reinforcement during timed reads so benchmark +iterations do not change the data they measure. + +The deterministic embedder makes the default run offline and repeatable. Latency remains +machine-dependent, so reports include the runtime/backend/corpus shape and never compare +numbers from unlike environments. The optional acceptance settings add cold/warm samples, +bounded concurrency, and independently-created worker processes without changing the small, +single-process default. + +Usage:: + + python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 + python -m eval.performance --dataset eval/datasets/codemem.jsonl --iterations 20 --json + python -m eval.performance --dataset cases.jsonl --concurrency 4 --processes 5 +""" +from __future__ import annotations + +import argparse +import json +import platform +import statistics +import sys +import time +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from engraphis.backends import DeterministicEmbedder, NumpyVectorIndex +from engraphis.backends.reranker import IdentityReranker +from engraphis.core.context import RegexTokenCounter +from engraphis.core.engine import MemoryEngine +from engraphis.core.interfaces import MemoryType, Scope, SearchFilter +from engraphis.core.store import Store +from eval import metrics +from eval.harness import load_dataset + + +SUPPORTED_CONCURRENCY = (1, 4, 16) + + +@dataclass(frozen=True) +class AcceptanceConfig: + """Acceptance-protocol controls, validated before an expensive run starts.""" + + concurrency: int = 1 + processes: int = 1 + minimum_queries: int = 0 + canonical: bool = False + + def validate(self, question_count: int) -> None: + if self.concurrency not in SUPPORTED_CONCURRENCY: + choices = ", ".join(str(value) for value in SUPPORTED_CONCURRENCY) + raise ValueError(f"concurrency must be one of: {choices}") + if self.processes < 1: + raise ValueError("processes must be at least 1") + if self.minimum_queries < 0: + raise ValueError("minimum_queries must be at least 0") + if question_count < self.minimum_queries: + raise ValueError( + f"dataset has {question_count} queries; minimum_queries requires " + f"at least {self.minimum_queries}" + ) + if self.canonical and question_count < 1000: + raise ValueError("canonical acceptance requires at least 1000 queries") + if self.canonical and self.processes < 5: + raise ValueError("canonical acceptance requires at least 5 processes") + + +@dataclass +class _Measurements: + cold_latencies_ms: list[float] + warm_latencies_ms: list[float] + context_tokens: list[int] + source_tokens: list[int] + full_payload_tokens: list[int] + compact_payload_tokens: list[int] + quality: list[dict] + + +def _percentile(values: list[float], percentile: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + position = (len(ordered) - 1) * percentile + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + fraction = position - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction + + +def _latency_summary(values: list[float]) -> dict[str, float]: + return { + "min": round(min(values, default=0.0), 3), + "p50": round(_percentile(values, 0.50), 3), + "p95": round(_percentile(values, 0.95), 3), + "p99": round(_percentile(values, 0.99), 3), + "max": round(max(values, default=0.0), 3), + } + + +def _question_count(dataset: list[dict]) -> int: + return sum(len(case.get("questions") or []) for case in dataset) + + +def _process_rss_bytes() -> Optional[int]: + """Best-effort peak RSS, normalized to bytes when the platform exposes it.""" + try: + import resource # Unix only; intentionally optional on Windows. + except ImportError: + return None + usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # macOS reports bytes, while Linux and the BSDs report KiB. + return int(usage if platform.system() == "Darwin" else usage * 1024) + + +def _storage_bytes(store: Store) -> Optional[int]: + """Report allocated SQLite storage even when the benchmark uses ``:memory:``.""" + try: + page_count = store.conn.execute("PRAGMA page_count").fetchone()[0] + page_size = store.conn.execute("PRAGMA page_size").fetchone()[0] + except Exception: # pragma: no cover - defensive for non-SQLite Store adapters. + return None + return int(page_count) * int(page_size) + + +def _serialized_tokens(payload: dict, counter: RegexTokenCounter) -> int: + """Count the exact canonical JSON payload used for compact/full comparison.""" + encoded = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return counter(encoded) + + +def _compact_provenance(value: object) -> dict: + """Match the bounded provenance identity emitted by compact recall responses.""" + if not isinstance(value, dict): + return {} + keys = ("source", "source_kind", "trusted", "kind", "origin") + return {key: value[key] for key in keys if key in value} + + +def _compact_payload(result) -> dict: + """Mirror ``engraphis_recall_context`` without serializing unpacked candidates.""" + by_id = {str(chunk.get("id") or ""): chunk for chunk in result.chunks} + sources = [] + for ordinal, packed in enumerate(result.packed_chunks, start=1): + chunk = by_id.get(str(packed.id or ""), {}) + source = { + "n": ordinal, + "id": packed.id, + "tokens": packed.tokens, + } + if chunk.get("title"): + source["title"] = chunk["title"] + provenance = _compact_provenance(chunk.get("provenance")) + if provenance: + source["provenance"] = provenance + if packed.truncated: + source["truncated"] = True + if packed.reason and packed.reason not in {"full", "summary"}: + source["reason"] = packed.reason + sources.append(source) + usage = vars(result.usage) if result.usage else {} + return {"context": result.context, "sources": sources, "usage": usage} + + +def _measure_recall( + engine: MemoryEngine, + question: dict, + search_filter: SearchFilter, + *, + k: int, + token_budget: int, +) -> tuple[dict, float]: + started = time.perf_counter_ns() + result = engine.recall_engine.recall( + question["q"], search_filter, k=k, reinforce=False, token_budget=token_budget + ) + return result, (time.perf_counter_ns() - started) / 1_000_000 + + +def _measure_batch( + engine: MemoryEngine, + questions: list[dict], + search_filter: SearchFilter, + *, + k: int, + token_budget: int, + concurrency: int, +) -> list[tuple[dict, float]]: + if concurrency == 1: + return [ + _measure_recall(engine, question, search_filter, k=k, token_budget=token_budget) + for question in questions + ] + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [ + executor.submit( + _measure_recall, + engine, + question, + search_filter, + k=k, + token_budget=token_budget, + ) + for question in questions + ] + return [future.result() for future in futures] + + +def _run_single( + dataset: list[dict], + *, + k: int, + dim: int, + warmups: int, + iterations: int, + filler_memories: int, + token_budget: int, + config: AcceptanceConfig, + process_number: int, + embedder: Optional[DeterministicEmbedder] = None, +) -> tuple[dict, _Measurements]: + embedder = embedder or DeterministicEmbedder(dim=dim) + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("performance") + repo_id = store.get_or_create_repo(workspace_id, "corpus") + index = NumpyVectorIndex(store) + engine = MemoryEngine(store, embedder, index, IdentityReranker()) + search_filter = SearchFilter( + workspace_id=workspace_id, + repo_id=repo_id, + include_ancestors=True, + ) + + id_to_tags: dict[str, list[str]] = {} + id_to_text: dict[str, str] = {} + questions: list[dict] = [] + for case_number, case in enumerate(dataset): + case_id = str(case.get("id") or f"case-{case_number}") + for memory_number, memory in enumerate(case.get("memories") or []): + memory_id = engine.remember( + memory["text"], + workspace_id=workspace_id, + repo_id=repo_id, + mtype=MemoryType.EPISODIC, + scope=Scope.REPO, + resolve_conflicts=False, + ) + tag = str(memory.get("tag") or f"memory-{memory_number}") + id_to_tags.setdefault(memory_id, []).append(f"{case_id}:{tag}") + id_to_text[memory_id] = memory["text"] + for question in case.get("questions") or []: + questions.append({ + "q": question["q"], + "answer": question.get("answer", ""), + "supporting": [ + f"{case_id}:{tag}" for tag in question.get("supporting", []) + ], + }) + + for number in range(filler_memories): + engine.remember( + "Synthetic benchmark filler %06d records unrelated deterministic context " + "for corpus scaling." % number, + workspace_id=workspace_id, + repo_id=repo_id, + mtype=MemoryType.EPISODIC, + scope=Scope.REPO, + resolve_conflicts=False, + ) + + cold = _measure_batch( + engine, + questions, + search_filter, + k=k, + token_budget=token_budget, + concurrency=config.concurrency, + ) + for _ in range(warmups): + _measure_batch( + engine, + questions, + search_filter, + k=k, + token_budget=token_budget, + concurrency=config.concurrency, + ) + + measurements = _Measurements([], [], [], [], [], [], []) + counter = RegexTokenCounter() + for iteration in range(iterations): + for question_number, (result, latency_ms) in enumerate(_measure_batch( + engine, + questions, + search_filter, + k=k, + token_budget=token_budget, + concurrency=config.concurrency, + )): + measurements.warm_latencies_ms.append(latency_ms) + if iteration != 0: + continue + retrieved_ids = [chunk["id"] for chunk in result.chunks] + retrieved_tags = [ + tag for memory_id in retrieved_ids for tag in id_to_tags.get(memory_id, []) + ] + retrieved_texts = [id_to_text.get(memory_id, "") for memory_id in retrieved_ids] + usage = result.usage + measurements.context_tokens.append(usage.context_tokens if usage else 0) + measurements.source_tokens.append(usage.source_tokens if usage else 0) + full_payload = {"context": result.context, "memories": result.chunks} + compact_payload = _compact_payload(result) + measurements.full_payload_tokens.append(_serialized_tokens(full_payload, counter)) + measurements.compact_payload_tokens.append( + _serialized_tokens(compact_payload, counter) + ) + question = questions[question_number] + measurements.quality.append({ + "question": question_number, + "recall_at_k": metrics.recall_at_k(retrieved_tags, question["supporting"]), + "hit_at_k": metrics.hit_at_k(retrieved_tags, question["supporting"]), + "answer_token_recall": metrics.answer_token_recall( + retrieved_texts, question["answer"] + ), + }) + + measurements.cold_latencies_ms = [latency_ms for _, latency_ms in cold] + process_resources = { + "process": process_number, + "rss_bytes": _process_rss_bytes(), + "storage_bytes": _storage_bytes(store), + } + corpus = { + "dataset_cases": len(dataset), + "memories": store.conn.execute("SELECT COUNT(*) AS n FROM memories").fetchone()["n"], + "questions": len(questions), + "filler_memories": filler_memories, + } + environment = { + "python": platform.python_version(), + "platform": platform.system().lower(), + "architecture": platform.machine().lower(), + "embedder": type(embedder).__name__, + "vector_backend": type(index).__name__, + } + store.close() + return {"corpus": corpus, "environment": environment, "resources": process_resources}, measurements + + +def _build_report( + base: dict, + measurements: list[_Measurements], + *, + k: int, + warmups: int, + iterations: int, + token_budget: int, + config: AcceptanceConfig, + question_count: int, + resources: list[dict], +) -> dict: + cold_latencies = [value for item in measurements for value in item.cold_latencies_ms] + warm_latencies = [value for item in measurements for value in item.warm_latencies_ms] + context_tokens = [value for item in measurements for value in item.context_tokens] + source_tokens = [value for item in measurements for value in item.source_tokens] + full_payload_tokens = [value for item in measurements for value in item.full_payload_tokens] + compact_payload_tokens = [value for item in measurements for value in item.compact_payload_tokens] + quality = [value for item in measurements for value in item.quality] + full_total = sum(full_payload_tokens) + compact_total = sum(compact_payload_tokens) + saved_total = full_total - compact_total + savings_ratios = [ + 1.0 - compact / max(1, full) + for full, compact in zip(full_payload_tokens, compact_payload_tokens) + ] + count = max(len(quality), 1) + rss_values = [item["rss_bytes"] for item in resources if item["rss_bytes"] is not None] + storage_values = [ + item["storage_bytes"] for item in resources if item["storage_bytes"] is not None + ] + warm_summary = _latency_summary(warm_latencies) + + return { + "schema": "engraphis-performance/v1", + "environment": base["environment"], + "corpus": base["corpus"], + "run": { + "k": k, + "warmups": warmups, + "iterations": iterations, + # Kept for compatibility: these are the warm, steady-state timed recalls. + "timed_recalls": len(warm_latencies), + "cold_timed_recalls": len(cold_latencies), + "warm_timed_recalls": len(warm_latencies), + "token_budget": token_budget, + }, + "acceptance": { + "concurrency": config.concurrency, + "independent_processes": config.processes, + "minimum_queries": config.minimum_queries, + "canonical": config.canonical, + "query_count": question_count, + "valid": True, + }, + "quality": { + "recall_at_k": round(sum(item["recall_at_k"] for item in quality) / count, 4), + "hit_at_k": round(sum(item["hit_at_k"] for item in quality) / count, 4), + "answer_token_recall": round( + sum(item["answer_token_recall"] for item in quality) / count, 4 + ), + }, + "context": { + "mean_tokens": round(sum(context_tokens) / max(len(context_tokens), 1), 2), + "max_tokens": max(context_tokens, default=0), + "mean_source_tokens": round( + sum(source_tokens) / max(len(source_tokens), 1), 2 + ), + "full_serialized_payload_tokens": full_total, + "compact_serialized_payload_tokens": compact_total, + "saved_serialized_payload_tokens": saved_total, + "serialized_payload_savings_ratio": round( + saved_total / max(1, full_total), 4 + ), + # Retain the original median measure for existing JSON consumers. + "median_serialized_payload_savings_ratio": round( + statistics.median(savings_ratios) if savings_ratios else 0.0, 4 + ), + "token_counter": "engraphis.regex.v1", + }, + "latency_ms": { + **warm_summary, + "cold": _latency_summary(cold_latencies), + "warm": warm_summary, + }, + "resources": { + "processes": resources, + "max_process_rss_bytes": max(rss_values, default=None), + "max_storage_bytes": max(storage_values, default=None), + }, + "detail": quality, + } + + +def run( + dataset: list[dict], + *, + k: int = 5, + dim: int = 256, + warmups: int = 1, + iterations: int = 5, + filler_memories: int = 0, + token_budget: int = 1500, + embedder: Optional[DeterministicEmbedder] = None, + concurrency: int = 1, + processes: int = 1, + minimum_queries: int = 0, + canonical: bool = False, +) -> dict: + """Benchmark recall and return a JSON-safe report. + + Existing callers keep the single-process deterministic path. ``processes`` creates + isolated in-memory corpora in child processes; a caller-provided embedder is therefore + intentionally limited to the established single-process API. + """ + k = max(1, int(k)) + warmups = max(0, int(warmups)) + iterations = max(1, int(iterations)) + filler_memories = max(0, int(filler_memories)) + token_budget = max(0, int(token_budget)) + if canonical: + raise ValueError("canonical acceptance requires run_acceptance_matrix") + config = AcceptanceConfig( + concurrency=int(concurrency), + processes=int(processes), + minimum_queries=int(minimum_queries), + canonical=False, + ) + question_count = _question_count(dataset) + config.validate(question_count) + if embedder is not None and config.processes != 1: + raise ValueError("a custom embedder is only supported with processes=1") + + if config.processes == 1: + base, measurement = _run_single( + dataset, + k=k, + dim=dim, + warmups=warmups, + iterations=iterations, + filler_memories=filler_memories, + token_budget=token_budget, + config=config, + process_number=0, + embedder=embedder, + ) + return _build_report( + base, + [measurement], + k=k, + warmups=warmups, + iterations=iterations, + token_budget=token_budget, + config=config, + question_count=question_count, + resources=[base["resources"]], + ) + + worker_args = { + "k": k, + "dim": dim, + "warmups": warmups, + "iterations": iterations, + "filler_memories": filler_memories, + "token_budget": token_budget, + "config": config, + } + with ProcessPoolExecutor(max_workers=config.processes) as executor: + futures = [ + executor.submit(_run_single, dataset, process_number=number, **worker_args) + for number in range(config.processes) + ] + process_results = [future.result() for future in futures] + base, _ = process_results[0] + return _build_report( + base, + [measurement for _, measurement in process_results], + k=k, + warmups=warmups, + iterations=iterations, + token_budget=token_budget, + config=config, + question_count=question_count, + resources=[result["resources"] for result, _ in process_results], + ) + + +def _canonical_matrix_concurrencies(concurrencies: Optional[list[int]]) -> tuple[int, ...]: + requested = tuple(SUPPORTED_CONCURRENCY if concurrencies is None else concurrencies) + if len(requested) != len(SUPPORTED_CONCURRENCY) or set(requested) != set( + SUPPORTED_CONCURRENCY + ): + choices = ", ".join(str(value) for value in SUPPORTED_CONCURRENCY) + raise ValueError(f"canonical acceptance matrix must include every concurrency: {choices}") + return SUPPORTED_CONCURRENCY + + +def run_acceptance_matrix( + dataset: list[dict], + *, + k: int = 5, + dim: int = 256, + warmups: int = 1, + iterations: int = 5, + filler_memories: int = 0, + token_budget: int = 1500, + processes: int = 5, + minimum_queries: int = 1000, + concurrencies: Optional[list[int]] = None, +) -> dict: + """Run the complete canonical 1/4/16-concurrency acceptance protocol. + + ``run`` remains the backwards-compatible per-slice primitive. This wrapper is + intentionally the only public API that declares a complete canonical result, and + validates every requirement before starting the expensive process matrix. + """ + matrix = _canonical_matrix_concurrencies(concurrencies) + effective_minimum = max(1000, int(minimum_queries)) + question_count = _question_count(dataset) + AcceptanceConfig( + concurrency=1, + processes=int(processes), + minimum_queries=effective_minimum, + canonical=True, + ).validate(question_count) + + slices = {} + for concurrency in matrix: + slices[str(concurrency)] = run( + dataset, + k=k, + dim=dim, + warmups=warmups, + iterations=iterations, + filler_memories=filler_memories, + token_budget=token_budget, + concurrency=concurrency, + processes=processes, + minimum_queries=effective_minimum, + canonical=False, + ) + return { + "schema": "engraphis-performance-matrix/v1", + "acceptance": { + "canonical": True, + "concurrency_matrix": list(matrix), + "independent_processes": int(processes), + "minimum_queries": effective_minimum, + "query_count": question_count, + "valid": True, + }, + "slices": slices, + } + + +def _print(report: dict) -> None: + corpus = report["corpus"] + run_info = report["run"] + quality = report["quality"] + latency = report["latency_ms"] + context = report["context"] + environment = report["environment"] + acceptance = report["acceptance"] + print( + "Engraphis performance — " + f"{corpus['memories']} memories · {corpus['questions']} questions · " + f"{run_info['timed_recalls']} warm timed recalls @ k={run_info['k']}" + ) + print( + " environment : " + f"{environment['platform']}/{environment['architecture']} · " + f"Python {environment['python']} · " + f"{environment['embedder']} + {environment['vector_backend']}" + ) + print( + " acceptance : " + f"concurrency={acceptance['concurrency']} · " + f"processes={acceptance['independent_processes']} · " + f"cold={run_info['cold_timed_recalls']} warm={run_info['warm_timed_recalls']}" + ) + print( + " quality : " + f"recall@k={quality['recall_at_k']:.3f} · " + f"hit@k={quality['hit_at_k']:.3f} · " + f"answer-token={quality['answer_token_recall']:.3f}" + ) + print( + " context tokens : " + f"mean={context['mean_tokens']:.2f} · max={context['max_tokens']} · " + "compact payload saved=" + f"{context['serialized_payload_savings_ratio']:.1%}" + ) + print( + " recall latency (ms) : " + f"cold p50={latency['cold']['p50']:.3f} · " + f"warm p50={latency['p50']:.3f} · p95={latency['p95']:.3f} · " + f"p99={latency['p99']:.3f} · max={latency['max']:.3f}" + ) + + +def _print_matrix(report: dict) -> None: + acceptance = report["acceptance"] + print( + "Engraphis canonical performance matrix — " + f"queries={acceptance['query_count']} · " + f"processes={acceptance['independent_processes']} · " + f"concurrency={acceptance['concurrency_matrix']}" + ) + for concurrency in acceptance["concurrency_matrix"]: + slice_report = report["slices"][str(concurrency)] + latency = slice_report["latency_ms"] + print( + f" concurrency={concurrency:<2} : " + f"cold p50={latency['cold']['p50']:.3f}ms · " + f"warm p50={latency['p50']:.3f}ms · p95={latency['p95']:.3f}ms · " + f"p99={latency['p99']:.3f}ms" + ) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Measure full-pipeline Engraphis recall quality, context, and latency." + ) + parser.add_argument( + "--dataset", + default=str(Path(__file__).resolve().parent / "datasets" / "codemem.jsonl"), + ) + parser.add_argument("--k", type=int, default=5) + parser.add_argument("--dim", type=int, default=256) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--token-budget", type=int, default=1500) + parser.add_argument( + "--filler-memories", + type=int, + default=0, + help="add deterministic unrelated memories to measure corpus scaling", + ) + parser.add_argument( + "--concurrency", + type=int, + choices=SUPPORTED_CONCURRENCY, + default=1, + help="concurrent queries per isolated process (default: 1)", + ) + parser.add_argument( + "--processes", + type=int, + default=1, + help="number of independent benchmark processes (default: 1)", + ) + parser.add_argument( + "--minimum-queries", + type=int, + default=0, + help="reject a dataset with fewer benchmark queries", + ) + parser.add_argument( + "--canonical", + action="store_true", + help="reserved for compatibility; use --acceptance-matrix for canonical protocol", + ) + parser.add_argument( + "--acceptance-matrix", + action="store_true", + help="run the canonical 1/4/16-concurrency, >=5-process acceptance matrix", + ) + parser.add_argument("--json", action="store_true", help="print the full JSON report") + args = parser.parse_args(argv) + dataset = load_dataset(args.dataset) + if args.acceptance_matrix: + report = run_acceptance_matrix( + dataset, + k=args.k, + dim=args.dim, + warmups=args.warmups, + iterations=args.iterations, + filler_memories=args.filler_memories, + token_budget=args.token_budget, + processes=args.processes, + minimum_queries=args.minimum_queries, + ) + else: + report = run( + dataset, + k=args.k, + dim=args.dim, + warmups=args.warmups, + iterations=args.iterations, + filler_memories=args.filler_memories, + token_budget=args.token_budget, + concurrency=args.concurrency, + processes=args.processes, + minimum_queries=args.minimum_queries, + canonical=args.canonical, + ) + if args.json: + print(json.dumps(report, indent=2)) + else: + (_print_matrix if args.acceptance_matrix else _print)(report) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/run_longmemeval_v2.py b/eval/run_longmemeval_v2.py new file mode 100644 index 00000000..7863befa --- /dev/null +++ b/eval/run_longmemeval_v2.py @@ -0,0 +1,110 @@ +"""Run the official LongMemEval-V2 harness with Engraphis registered. + +The upstream harness uses an import-time memory registry and does not discover +third-party backends. This entry point registers Engraphis first, then delegates +the unchanged command line to ``evaluation.harness``. +""" +from __future__ import annotations + +import importlib +import runpy +import subprocess +from pathlib import Path +from typing import Callable + + +PINNED_LONGMEMEVAL_V2_REVISION = "6f020ac2fc3275e46c706d3406e02c3ed79b7be2" +PINNED_READER_MODEL = "Qwen/Qwen3.5-9B" +PINNED_READER_REVISION = "c202236235762e1c871ad0ccb60c8ee5ba337b9a" + + +def verify_official_checkout(memory_module: object) -> None: + """Require the exact audited upstream revision before delegating a run. + + ``PYTHONPATH`` alone is not provenance: it can point at an arbitrary local + fork with a compatible registry. The wrapper is reserved for official V2 + execution, so fail before any data/model work if the imported memory module + is not inside the pinned upstream checkout. + """ + module_path = getattr(memory_module, "__file__", None) + if not isinstance(module_path, str) or not module_path: + raise SystemExit("LongMemEval-V2 memory module has no verifiable source path.") + try: + root = subprocess.check_output( + ["git", "-C", str(Path(module_path).resolve().parent), "rev-parse", "--show-toplevel"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + revision = subprocess.check_output( + ["git", "-C", root, "rev-parse", "HEAD"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except (OSError, subprocess.CalledProcessError) as exc: + raise SystemExit( + "LongMemEval-V2 must be an exact pinned Git checkout; could not verify its revision." + ) from exc + if revision != PINNED_LONGMEMEVAL_V2_REVISION: + raise SystemExit( + "LongMemEval-V2 checkout revision mismatch: expected " + f"{PINNED_LONGMEMEVAL_V2_REVISION}, found {revision or 'unknown'}." + ) + + +def pin_official_reader_processor() -> Callable[[], None]: + """Force the official harness reader processor onto the audited revision. + + The pinned upstream harness loads ``AutoProcessor`` without a revision. + This canonical Engraphis wrapper replaces that mutable default for the + complete delegated run and restores the global method afterwards. + """ + try: + from transformers import AutoProcessor + except ImportError as exc: # pragma: no cover - optional official-run dependency + raise SystemExit( + "canonical LongMemEval-V2 execution requires transformers and the pinned reader processor." + ) from exc + original = AutoProcessor.from_pretrained + + @classmethod + def pinned_from_pretrained( + cls: object, pretrained_model_name_or_path: object, *args: object, **kwargs: object + ) -> object: + del cls + if str(pretrained_model_name_or_path) != PINNED_READER_MODEL: + raise RuntimeError( + "canonical LongMemEval-V2 execution permits only the configured reader processor" + ) + requested_revision = kwargs.get("revision") + if requested_revision not in (None, PINNED_READER_REVISION): + raise RuntimeError("official harness requested a reader revision outside the canonical profile") + kwargs["revision"] = PINNED_READER_REVISION + return original(pretrained_model_name_or_path, *args, **kwargs) + + AutoProcessor.from_pretrained = pinned_from_pretrained + + def restore() -> None: + AutoProcessor.from_pretrained = original + + return restore + + +def main() -> None: + try: + memory_module = importlib.import_module("memory_modules.memory") + except ModuleNotFoundError as exc: + raise SystemExit( + "LongMemEval-V2 is not importable. Add the pinned official checkout " + "to PYTHONPATH before running this module." + ) from exc + verify_official_checkout(memory_module) + importlib.import_module("eval.longmemeval_v2") + restore_processor = pin_official_reader_processor() + try: + runpy.run_module("evaluation.harness", run_name="__main__") + finally: + restore_processor() + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 5a9be88f..f10fba0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,7 +184,7 @@ engraphis-update = "scripts.update:main" include-package-data = false [tool.setuptools.packages.find] -include = ["engraphis*", "scripts*"] +include = ["engraphis*", "scripts*", "eval*"] [tool.setuptools.package-data] # Keep the shipped dashboard assets explicit. A recursive catch-all also packages ignored @@ -194,6 +194,7 @@ include = ["engraphis*", "scripts*"] "engraphis.classic_assets" = ["*.html", "*.css", "*.js", "*.png", "*.ico", "vendor/*", "vendor/**/*"] "engraphis.dashboard_assets" = ["*.html", "*.css", "*.js", "*.png", "*.ico", "vendor/*", "vendor/**/*"] "engraphis" = ["commercial_manifest.json"] +"eval" = ["BASELINES.md", "configs/*.json", "datasets/*.jsonl"] [tool.setuptools.exclude-package-data] "*" = ["*.pyc", "*.pyo", "__pycache__/*"] diff --git a/scripts/release_evidence.py b/scripts/release_evidence.py new file mode 100644 index 00000000..0395940a --- /dev/null +++ b/scripts/release_evidence.py @@ -0,0 +1,369 @@ +"""Create a deterministic, content-safe manifest for a public release candidate. + +The evidence is deliberately limited to files and commands in this repository. It is +not an operational attestation for the hosted control plane, payment provider, or a +customer deployment. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +from pathlib import Path +from typing import Any, Iterable, Optional + +try: # Python 3.11+ + import tomllib +except ImportError: # pragma: no cover - supported Python 3.9/3.10 + tomllib = None + + +FORMAT = "engraphis-release-evidence/2" +PACKAGE = "engraphis" +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") +_COMMIT = re.compile(r"[0-9a-f]{40}\Z") +_TAG = re.compile(r"v([0-9]+\.[0-9]+\.[0-9]+)\Z") +_SAFE_PATH = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*\Z") +_SECRET_NAME = re.compile( + r"(?:secret|token|password|credential|api[-_]?key|private[-_]?key)", re.IGNORECASE +) +_SECRET_VALUE = re.compile( + r"(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|\b(?:sk|rk|pk)_[A-Za-z0-9_-]{16,}\b|" + r"\bgh[pous]_[A-Za-z0-9_]{16,}\b|\bgithub_pat_[A-Za-z0-9_]{16,}\b|" + r"\bAKIA[0-9A-Z]{16}\b|\bengr_(?:ct|rt|at)_[A-Za-z0-9_-]{12,}\b)", + re.IGNORECASE, +) + + +class EvidenceError(ValueError): + """A release-evidence input is malformed, incomplete, or unsafe to publish.""" + + +def canonical_json_bytes(value: Any) -> bytes: + """Return one stable UTF-8 encoding suitable for a reproducible artifact.""" + return (json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n").encode( + "utf-8" + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _relative_path(root: Path, path: Path) -> str: + try: + relative = path.resolve().relative_to(root.resolve()).as_posix() + except ValueError as exc: + raise EvidenceError("evidence inputs must stay within the repository") from exc + if not _SAFE_PATH.fullmatch(relative) or _SECRET_NAME.search(relative): + raise EvidenceError("evidence input path is unsafe to publish") + return relative + + +def _reject_secret_like(value: Any) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise EvidenceError("evidence object keys must be strings") + if _SECRET_NAME.search(key): + raise EvidenceError("evidence must not include secret-like fields") + _reject_secret_like(item) + elif isinstance(value, (list, tuple)): + for item in value: + _reject_secret_like(item) + elif isinstance(value, str) and _SECRET_VALUE.search(value): + raise EvidenceError("evidence must not include secret-like values") + + +def _file_input(root: Path, relative: str) -> dict[str, str]: + path = root / relative + if not path.is_file(): + raise EvidenceError("required release input is missing: %s" % relative) + return {"path": _relative_path(root, path), "sha256": _sha256(path)} + + +def project_version(root: Path) -> str: + pyproject = root / "pyproject.toml" + try: + raw = pyproject.read_text(encoding="utf-8") + if tomllib is not None: + version = tomllib.loads(raw)["project"]["version"] + else: + project = re.search(r"(?ms)^\[project\]\s*(.*?)(?=^\[|\Z)", raw) + match = ( + re.search(r'(?m)^version\s*=\s*"([^"]+)"\s*$', project.group(1)) + if project else None + ) + if match is None: + raise KeyError("project.version") + version = match.group(1) + except (KeyError, OSError, ValueError) as exc: + raise EvidenceError("pyproject project.version is required") from exc + if not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise EvidenceError("project.version must use stable semantic version syntax") + return version + + +def git_commit(root: Path) -> str: + try: + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True, stderr=subprocess.DEVNULL + ).strip() + except (OSError, subprocess.CalledProcessError) as exc: + raise EvidenceError("could not determine the release commit") from exc + return validate_commit(commit) + + +def validate_commit(commit: str) -> str: + if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): + raise EvidenceError("release commit must be a lowercase 40-character SHA-1") + return commit + + +def validate_tag(tag: str, version: str) -> str: + """Require a canonical release tag that exactly names the package version.""" + if not isinstance(tag, str): + raise EvidenceError("release tag must be a stable semantic version tag") + match = _TAG.fullmatch(tag) + if match is None or match.group(1) != version: + raise EvidenceError("release tag must exactly match the package version") + return tag + + +def distribution_artifacts(directory: Path, version: str) -> list[dict[str, Any]]: + if not directory.is_dir(): + raise EvidenceError("distribution directory is missing") + allowed = (".whl", ".tar.gz") + paths = sorted(path for path in directory.iterdir() if path.is_file()) + if not paths: + raise EvidenceError("distribution directory is empty") + artifacts = [] + for path in paths: + name = path.name + if not name.endswith(allowed) or not _SAFE_PATH.fullmatch(name) or _SECRET_NAME.search(name): + raise EvidenceError("distribution directory contains an unsafe non-package file") + if not name.startswith(PACKAGE + "-" + version + ".") and not name.startswith( + PACKAGE + "-" + version + "-" + ): + raise EvidenceError("distribution filename does not match package version") + artifacts.append({"filename": name, "bytes": path.stat().st_size, "sha256": _sha256(path)}) + return artifacts + + +def sbom_artifact(root: Path, path: Path) -> dict[str, Any]: + """Validate and fingerprint the generated CycloneDX SBOM before publishing it.""" + if not path.is_file(): + raise EvidenceError("SBOM is missing") + relative = _relative_path(root, path) + if not path.name.endswith(".cdx.json"): + raise EvidenceError("SBOM filename must use the .cdx.json suffix") + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvidenceError("SBOM must be valid UTF-8 JSON") from exc + if not isinstance(parsed, dict) or parsed.get("bomFormat") != "CycloneDX": + raise EvidenceError("SBOM must be a CycloneDX JSON document") + if not isinstance(parsed.get("specVersion"), str) or not isinstance(parsed.get("components"), list): + raise EvidenceError("SBOM is missing required CycloneDX fields") + _reject_secret_like(parsed) + return { + "format": "CycloneDX", + "spec_version": parsed["specVersion"], + "filename": path.name, + "path": relative, + "bytes": path.stat().st_size, + "sha256": _sha256(path), + } + + +def check_manifest(root: Path) -> dict[str, list[dict[str, Any]]]: + """Return the exact public checks represented by this evidence format.""" + return { + "tests": [ + {"id": "ruff", "command": ["ruff", "check", "."], "inputs": []}, + { + "id": "pytest", + "command": ["python", "-m", "pytest", "-o", "addopts=", "tests/", "-q", "-rs"], + "inputs": [], + }, + { + "id": "privacy-boundary", + "command": [ + "python", "-m", "pytest", "-o", "addopts=", + "tests/test_public_research_boundary.py", "-q", + ], + "inputs": [], + }, + { + "id": "token-efficiency", + "command": [ + "python", "-m", "pytest", "-o", "addopts=", + "tests/test_compact_recall.py", "tests/test_eval_performance.py", "-q", + ], + "inputs": [], + }, + { + "id": "benchmark-schema-evidence", + "command": [ + "python", "-m", "pytest", "-o", "addopts=", + "tests/test_eval_harness.py", "tests/test_benchmark_evidence.py", "-q", + ], + "inputs": [], + }, + { + "id": "browser-e2e", + "command": ["npm", "run", "test:e2e"], + "workflow_job": "browser-accessibility", + "inputs": [], + }, + { + "id": "dependency-audit", + "command": ["python", "-m", "pip_audit", "--local"], + "inputs": [], + }, + { + "id": "container-smoke", + "command": ["docker", "build", "-t", "engraphis:release", "."], + "workflow_job": "docker-smoke", + "workflow_steps": [ + "Verify production image OCR runtime", + "Audit production image dependencies", + "Run customer-mode readiness smoke", + ], + "inputs": [], + }, + ], + "evaluations": [ + { + "id": "retrieval-sample", + "command": [ + "python", "-m", "eval.harness", "--dataset", "eval/datasets/sample.jsonl", "--k", "5" + ], + "inputs": [_file_input(root, "eval/datasets/sample.jsonl")], + }, + { + "id": "retrieval-codemem", + "command": [ + "python", "-m", "eval.harness", "--dataset", "eval/datasets/codemem.jsonl", "--k", "5" + ], + "inputs": [_file_input(root, "eval/datasets/codemem.jsonl")], + }, + { + "id": "retrieval-ablation", + "command": ["python", "-m", "eval.ablation"], + "inputs": [ + _file_input(root, "eval/datasets/sample.jsonl"), + _file_input(root, "eval/datasets/graph_multihop.jsonl"), + ], + }, + ], + } + + +def _verified_check_ids(manifest: dict[str, list[dict[str, Any]]]) -> set[str]: + return {check["id"] for group in manifest.values() for check in group} + + +def build_evidence( + root: Path, + distribution_directory: Path, + *, + commit: str, + tag: str, + sbom: Path, + verified_checks: Iterable[str] = (), +) -> dict[str, Any]: + """Build deterministic evidence; callers state which fixed checks they ran.""" + root = root.resolve() + version = project_version(root) + manifest = check_manifest(root) + expected = _verified_check_ids(manifest) + verified = sorted(set(verified_checks)) + if any(not isinstance(item, str) for item in verified) or set(verified) != expected: + missing = sorted(expected - set(verified)) + unexpected = sorted(set(verified) - expected) + details = [] + if missing: + details.append("missing=" + ",".join(missing)) + if unexpected: + details.append("unexpected=" + ",".join(unexpected)) + raise EvidenceError("verified checks must exactly match the public manifest (" + "; ".join(details) + ")") + checked_commit = validate_commit(commit) + checked_tag = validate_tag(tag, version) + evidence = { + "format": FORMAT, + "package": {"name": PACKAGE, "version": version}, + "commit": checked_commit, + "tag": checked_tag, + "provenance": { + "source": {"commit": checked_commit, "tag": checked_tag}, + "builder": { + "workflow": ".github/workflows/release.yml", + "job": "release-evidence", + "completed_gate_jobs": [ + "build", "python-matrix", "browser-accessibility", "docker-smoke", + ], + "sbom_generator": { + "name": "cyclonedx-bom", + "version": "7.3.0", + "command": [ + "cyclonedx-py", "environment", "--output-reproducible", "--of", "JSON", + "--pyproject", "pyproject.toml", + ], + }, + }, + }, + "source_inputs": [ + _file_input(root, "pyproject.toml"), + _file_input(root, "LICENSE"), + _file_input(root, "NOTICE"), + ], + "artifacts": distribution_artifacts(distribution_directory, version), + "sbom": sbom_artifact(root, sbom), + "checks": manifest, + "verified_checks": verified, + "limitations": [ + "This evidence attests only to the named source inputs, distributions, SBOM, and checks.", + "It does not attest to publication, release hosting, hosted services, payments, deployments, or runtime data.", + "The SBOM describes the Python environment used for this build; it is not an operating-system or container SBOM.", + ], + } + _reject_secret_like(evidence) + return evidence + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dist", type=Path, required=True, help="directory containing wheel and sdist") + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--commit", help="release commit; defaults to git HEAD") + parser.add_argument("--tag", required=True, help="release tag matching pyproject project.version") + parser.add_argument("--sbom", type=Path, required=True, help="generated CycloneDX JSON SBOM") + parser.add_argument("--verified-check", action="append", default=[], help="one completed public check id") + parser.add_argument("--output", type=Path, help="write canonical JSON instead of stdout") + args = parser.parse_args(argv) + try: + root = args.root.resolve() + evidence = build_evidence( + root, args.dist.resolve(), commit=args.commit or git_commit(root), + tag=args.tag, sbom=args.sbom.resolve(), + verified_checks=args.verified_check, + ) + encoded = canonical_json_bytes(evidence) + if args.output: + args.output.write_bytes(encoded) + else: + __import__("sys").stdout.buffer.write(encoded) + except EvidenceError as exc: + parser.error(str(exc)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_benchmark_adversarial.py b/tests/test_benchmark_adversarial.py new file mode 100644 index 00000000..692d8da0 --- /dev/null +++ b/tests/test_benchmark_adversarial.py @@ -0,0 +1,16 @@ +from pathlib import Path + +from eval.harness import load_dataset, run + + +def test_adversarial_fixture_keeps_off_topic_queries_out_of_retrieval_denominator(): + path = Path(__file__).resolve().parents[1] / "eval" / "datasets" / "adversarial.jsonl" + report = run(load_dataset(str(path)), k=2) + assert report["questions"] == 2 + assert report["scored_questions"] == 1 + assert report["recall_at_k"] == 1.0 + assert report["exclusions"] == [{ + "question_id": "sourdough-off-topic", + "reason": "off_topic_no_gold_evidence", + "detail": "", + }] diff --git a/tests/test_benchmark_evidence.py b/tests/test_benchmark_evidence.py new file mode 100644 index 00000000..099eff73 --- /dev/null +++ b/tests/test_benchmark_evidence.py @@ -0,0 +1,650 @@ +import json +from copy import deepcopy + +import pytest + +from eval import metrics +from eval.benchmark import ( + SCHEMA, + CANONICAL_TOKEN_BUDGETS, + LONGMEMEVAL_V2_CANONICAL_PROFILE_TEMPLATE, + canonical_benchmark_config, + count_tokens, + fixed_budget_curve, + paired_bootstrap_ci, + main, + question_record, + report_envelope, + stratified_bootstrap_ci, + validate_report, + write_canonical_artifact, +) + + +class CharacterTokenizer: + def encode(self, text): + return list(text) + + +def _complete_canonical_report(dataset, config): + """Minimal but fully auditable canonical envelope for validator coverage.""" + profile = config["canonical_profile"] + tokenizer_identity = ( + f"{profile['reader']['model']}@{profile['reader']['revision']}" + ) + record = question_record( + "q1", category="state", context_tokens=3, latency_ms=1.25, + retrieved_ids=["support"], supporting_ids=["support"], + recall_at_1=1.0, recall_at_5=1.0, recall_at_10=1.0, + mrr_at_1=1.0, mrr_at_5=1.0, mrr_at_10=1.0, + ndcg_at_1=1.0, ndcg_at_5=1.0, ndcg_at_10=1.0, + usage={ + "budget_tokens": config.get("token_budget") or 3, + "context_tokens": 3, + "token_counter": tokenizer_identity, + }, + ) + record["question_sha256"] = "a" * 64 + record["context_token_method"] = "pinned_reader_content_tokenizer" + record["context_tokenizer_identity"] = tokenizer_identity + rank_metrics = { + f"{metric}_at_{depth}": 1.0 + for metric in ("recall", "mrr", "ndcg") + for depth in (1, 5, 10) + } + curve_record = { + "question_id": "q1", + "excluded": False, + "context_tokens": 3, + "context_token_method": "pinned_reader_content_tokenizer", + "context_tokenizer_identity": tokenizer_identity, + "retrieved_ids": ["support"], + "supporting_ids": ["support"], + **rank_metrics, + } + report = report_envelope( + suite="fixture", dataset_path=dataset, config=config, records=[record], + metrics={ + **rank_metrics, + "confidence_intervals": { + field: { + "point": 1.0, + "low": 1.0, + "high": 1.0, + "n": 1, + "seed": 20260729, + "iterations": 1, + "strata_key": "category", + } + for field in rank_metrics + }, + "paired_bootstrap": { + "available": False, + "reason": "baseline_records_not_supplied", + "n": 0, + "delta": None, + "low": None, + "high": None, + "iterations": 1, + }, + "grounded_f1": {"available": False, "reason": "not_measured"}, + "abstention_f1": {"available": False, "reason": "not_measured"}, + "fixed_budget_curve": { + "available": True, + "rows": [{ + "token_budget": budget, + "status": "measured", + "n_total": 1, + "n_scored": 1, + "records": [dict(curve_record)], + **rank_metrics, + } for budget in CANONICAL_TOKEN_BUDGETS], + }, + }, + git_commit="a" * 40, + ) + report["models"] = {"embedder": { + "name": "FixtureEmbedder", + "model_id": profile["embedding"]["model"], + "revision": profile["embedding"]["revision"], + "sha256": "b" * 64, + }} + report["protocol"]["complete_dataset"] = True + report["protocol"]["source_questions"] = len(report["records"]) + return report + + +def test_metrics_cover_rank_sensitive_retrieval_quality(): + retrieved = ["noise", "evidence-a", "evidence-b"] + supporting = ["evidence-a", "evidence-b"] + assert metrics.mrr_at_k(retrieved, supporting, 3) == 0.5 + assert metrics.ndcg_at_k(retrieved, supporting, 3) > 0.6 + assert metrics.recall_at_k(retrieved[:1], supporting) == 0.0 + assert metrics.hit_at_k(retrieved[:1], supporting) == 0.0 + bundle = metrics.retrieval_metrics_at_depths(retrieved, supporting) + assert bundle["recall_at_1"] == 0.0 + assert bundle["recall_at_5"] == 1.0 + assert bundle["mrr_at_5"] == 0.5 + + +def test_envelope_hashes_dataset_config_and_retains_exclusions(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + excluded = {"question_id": "q2", "reason": "no_gold_evidence", "detail": ""} + records = [ + question_record("q1", category="state", supporting_ids=["m1"]), + question_record("q2", category="abstention", excluded=excluded), + ] + report = report_envelope( + suite="fixture", dataset_path=dataset, config={"k": 5}, records=records, + metrics={"recall": 1.0}, git_commit="abc123", + ) + assert report["schema"] == SCHEMA + assert report["suite"]["sha256"] + assert report["system"]["config_sha256"] + assert report["protocol"] == {"config": {"k": 5}, "n_total": 2, "n_scored": 1} + assert report["exclusions"] == [excluded] + assert json.loads(json.dumps(report))["schema"] == SCHEMA + + +def test_canonical_profile_validator_and_immutable_artifact_writer(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + profile = json.loads(json.dumps(LONGMEMEVAL_V2_CANONICAL_PROFILE_TEMPLATE)) + profile["benchmark"]["repository_revision"] = "a" * 40 + profile["benchmark"]["dataset_revision"] = "b" * 40 + profile["reader"]["revision"] = "c" * 40 + profile["embedding"]["revision"] = "d" * 40 + profile["baseline_label"] = "full_hybrid" + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid", profile=profile + ) + report = _complete_canonical_report(dataset, config) + assert validate_report(report, canonical=True) == [] + artifact = tmp_path / "artifacts" / "run.json" + written = write_canonical_artifact(report, artifact, canonical=True) + assert written["sha256"] in artifact.with_name("run.json.sha256").read_text("ascii") + assert json.loads(artifact.read_text("utf-8"))["schema"] == SCHEMA + assert write_canonical_artifact(report, artifact, canonical=True) == written + changed = dict(report) + changed["records"] = [dict(report["records"][0])] + changed["records"][0]["latency_ms"] = 2.0 + with pytest.raises(FileExistsError): + write_canonical_artifact(changed, artifact, canonical=True) + + +def test_report_validator_recomputes_embedded_config_digest(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + report = report_envelope( + suite="fixture", dataset_path=dataset, config={"baseline_label": "full_hybrid"}, + records=[question_record("q1")], git_commit="abc123", + ) + report["protocol"]["config"]["baseline_label"] = "dense_only" + + errors = validate_report(report) + + assert "system.config_sha256 must match the canonical protocol.config digest" in errors + + +def test_report_validator_rejects_inconsistent_or_duplicate_exclusions(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + excluded = {"question_id": "q2", "reason": "no_gold_evidence", "detail": ""} + report = report_envelope( + suite="fixture", dataset_path=dataset, config={"k": 5}, + records=[ + question_record("q1"), + question_record("q2", excluded=excluded), + ], + git_commit="abc123", + ) + assert validate_report(report) == [] + + report["exclusions"] = [excluded, excluded] + errors = validate_report(report) + assert "exclusion question_id values must be unique" in errors + + report["exclusions"] = [] + errors = validate_report(report) + assert "top-level exclusions must exactly match per-record exclusions" in errors + + +def test_default_canonical_profile_is_pinned_and_rejects_mutable_revisions(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + report = _complete_canonical_report(dataset, config) + assert validate_report(report, canonical=True) == [] + assert all( + len(value) == 40 + for value in ( + config["canonical_profile"]["benchmark"]["repository_revision"], + config["canonical_profile"]["benchmark"]["dataset_revision"], + config["canonical_profile"]["reader"]["revision"], + config["canonical_profile"]["embedding"]["revision"], + ) + ) + assert config["token_budgets"] == list(CANONICAL_TOKEN_BUDGETS) + + config["canonical_profile"]["reader"]["revision"] = "main" + errors = validate_report(report, canonical=True) + assert any("reader.revision" in error and "immutable" in error for error in errors) + + +def test_canonical_validator_rejects_unpinned_commit_raw_queries_and_unlabeled_measurements(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + report = _complete_canonical_report(dataset, config) + report["system"]["git_commit"] = "not-a-commit" + report["records"][0]["q"] = "private source question" + report["records"][0].pop("context_token_method") + report["metrics"].pop("recall_at_10") + + errors = validate_report(report, canonical=True) + + assert any("git_commit" in error for error in errors) + assert "canonical records must not contain raw query text" in errors + assert any("context_token_method" in error for error in errors) + assert any("metrics.recall_at_10" in error for error in errors) + + config["canonical_profile"]["reader"]["revision"] = "C" * 40 + errors = validate_report(report, canonical=True) + assert any("reader.revision" in error and "immutable" in error for error in errors) + + +def test_canonical_validator_requires_grounded_metrics_or_explicit_unavailability(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + report = _complete_canonical_report(dataset, config) + report["metrics"].pop("grounded_f1") + report["metrics"]["abstention_f1"] = {"available": False} + + errors = validate_report(report, canonical=True) + + assert any("grounded_f1" in error and "unavailable reason" in error for error in errors) + assert any("abstention_f1" in error and "unavailable reason" in error for error in errors) + + +def test_canonical_validator_requires_measured_rows_for_every_fixed_budget(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + report = _complete_canonical_report(dataset, config) + report["metrics"]["fixed_budget_curve"]["rows"].pop() + + errors = validate_report(report, canonical=True) + + assert "canonical fixed-budget curve must contain every canonical token budget" in errors + report["metrics"]["fixed_budget_curve"] = {"available": False, "reason": "not_run"} + errors = validate_report(report, canonical=True) + assert "canonical fixed-budget curve is unavailable and cannot qualify as evidence" in errors + + report = _complete_canonical_report(dataset, config) + report["metrics"]["fixed_budget_curve"]["rows"][0]["records"][0]["excluded"] = True + errors = validate_report(report, canonical=True) + assert "canonical fixed-budget curve 256 records must preserve exclusion state" in errors + + +def test_canonical_validator_requires_complete_dataset_cardinality(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + valid = _complete_canonical_report(dataset, config) + assert validate_report(valid, canonical=True) == [] + + missing_complete = deepcopy(valid) + missing_complete["protocol"].pop("complete_dataset") + assert "canonical protocol.complete_dataset must be true" in validate_report( + missing_complete, canonical=True + ) + + for invalid_count in (True, 0, 2): + mismatched = deepcopy(valid) + mismatched["protocol"]["source_questions"] = invalid_count + errors = validate_report(mismatched, canonical=True) + assert any("protocol.source_questions" in error for error in errors) + + +def test_canonical_validator_rejects_invalid_numeric_and_token_accounting(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + config["token_budget"] = 4 + valid = _complete_canonical_report(dataset, config) + valid["records"][0]["usage"] = { + "budget_tokens": 4, + "context_tokens": 3, + "token_counter": valid["records"][0]["context_tokenizer_identity"], + } + assert validate_report(valid, canonical=True) == [] + + mutations = ( + (("metrics", "recall_at_1"), True, "metrics.recall_at_1"), + (("records", 0, "recall_at_1"), True, "records require recall_at_1"), + (("records", 0, "latency_ms"), float("inf"), "latency_ms"), + (("records", 0, "context_tokens"), float("nan"), "context_tokens"), + (("records", 0, "context_tokens"), -1, "context_tokens"), + (("records", 0, "context_tokens"), 5, "must not exceed protocol token_budget"), + ( + ("records", 0, "usage", "context_tokens"), + 5, + "usage.context_tokens must not exceed usage.budget_tokens", + ), + ( + ("records", 0, "usage", "budget_tokens"), + 5, + "usage.budget_tokens must equal protocol token_budget", + ), + ( + ("records", 0, "usage", "source_tokens"), + True, + "usage.source_tokens must be non-negative and finite", + ), + ( + ("records", 0, "usage", "savings_ratio"), + float("inf"), + "usage.savings_ratio must be a number in [0, 1]", + ), + ( + ("metrics", "fixed_budget_curve", "rows", 0, "recall_at_1"), + True, + "fixed-budget curve 256 requires recall_at_1", + ), + ( + ("metrics", "fixed_budget_curve", "rows", 0, "records", 0, "context_tokens"), + 257, + "context_tokens within budget", + ), + ) + for path, value, expected in mutations: + report = deepcopy(valid) + target = report + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + errors = validate_report(report, canonical=True) + assert any(expected in error for error in errors), (path, errors) + + +def test_canonical_validator_rejects_tampered_confidence_intervals(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + valid = _complete_canonical_report(dataset, config) + assert validate_report(valid, canonical=True) == [] + + mutations = ( + ("point", float("nan"), "point/low/high must be finite"), + ("low", -0.1, "point/low/high must be finite"), + ("high", 1.1, "point/low/high must be finite"), + ("high", 0.5, "low <= point <= high"), + ("point", 0.5, ".point must match metrics.recall_at_1"), + ("n", 2, ".n must equal the non-excluded record count"), + ("seed", -1, ".seed must be a non-negative integer"), + ("iterations", 0, ".iterations must be a positive integer"), + ("iterations", -1, ".iterations must be a positive integer"), + ("iterations", True, ".iterations must be a positive integer"), + ("strata_key", "topic", ".strata_key must equal category"), + ) + for key, value, expected in mutations: + report = deepcopy(valid) + report["metrics"]["confidence_intervals"]["recall_at_1"][key] = value + errors = validate_report(report, canonical=True) + assert any(expected in error for error in errors), (key, value, errors) + + extra = deepcopy(valid) + extra["metrics"]["confidence_intervals"]["recall_at_1"]["mean"] = 1.0 + errors = validate_report(extra, canonical=True) + assert any("must match the canonical confidence interval schema" in error for error in errors) + + missing = deepcopy(valid) + missing["metrics"]["confidence_intervals"].pop("recall_at_1") + errors = validate_report(missing, canonical=True) + assert ( + "canonical metrics.confidence_intervals must exactly cover every rank metric" + in errors + ) + + +def test_canonical_validator_rejects_tampered_paired_bootstrap_payloads(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + valid = _complete_canonical_report(dataset, config) + + unavailable_mutations = ( + ("reason", "", ".reason must be a non-empty string"), + ("n", 1, ".n must be zero when unavailable"), + ("delta", 0.0, "delta/low/high must be null when unavailable"), + ("iterations", 0, ".iterations must be a positive integer"), + ("iterations", True, ".iterations must be a positive integer"), + ) + for key, value, expected in unavailable_mutations: + report = deepcopy(valid) + report["metrics"]["paired_bootstrap"][key] = value + errors = validate_report(report, canonical=True) + assert any(expected in error for error in errors), (key, value, errors) + + available = deepcopy(valid) + available["metrics"]["paired_bootstrap"] = { + "available": True, + "metric": "recall_at_5", + "delta": 0.25, + "low": 0.0, + "high": 0.5, + "n": 1, + "seed": 20260729, + "iterations": 20, + } + assert validate_report(available, canonical=True) == [] + + available_mutations = ( + ("metric", "recall_at_3", ".metric must name a canonical rank metric"), + ("delta", float("inf"), "delta/low/high must be finite"), + ("delta", 1.1, "delta/low/high must be finite"), + ("low", -1.1, "delta/low/high must be finite"), + ("high", 1.1, "delta/low/high must be finite"), + ("low", 0.3, "low <= delta <= high"), + ("n", 0, ".n must equal the positive non-excluded record count"), + ("n", 2, ".n must equal the positive non-excluded record count"), + ("seed", -1, ".seed must be a non-negative integer"), + ("iterations", 0, ".iterations must be a positive integer"), + ("iterations", True, ".iterations must be a positive integer"), + ) + for key, value, expected in available_mutations: + report = deepcopy(available) + report["metrics"]["paired_bootstrap"][key] = value + errors = validate_report(report, canonical=True) + assert any(expected in error for error in errors), (key, value, errors) + + extra = deepcopy(available) + extra["metrics"]["paired_bootstrap"]["mean"] = 0.25 + errors = validate_report(extra, canonical=True) + assert any("available payload must match the canonical schema" in error for error in errors) + + +def test_canonical_validator_recomputes_all_rank_aggregates_from_record_ids(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + valid = _complete_canonical_report(dataset, config) + + top_level = deepcopy(valid) + top_level["metrics"]["recall_at_5"] = 0.5 + errors = validate_report(top_level, canonical=True) + assert ( + "canonical metrics.recall_at_5 must equal the non-excluded record mean" + in errors + ) + + curve_aggregate = deepcopy(valid) + curve_aggregate["metrics"]["fixed_budget_curve"]["rows"][0]["ndcg_at_10"] = 0.5 + errors = validate_report(curve_aggregate, canonical=True) + assert any( + "fixed-budget curve 256 ndcg_at_10" in error + and "non-excluded record mean" in error + for error in errors + ) + + curve_measurement = deepcopy(valid) + measurement = curve_measurement["metrics"]["fixed_budget_curve"]["rows"][0]["records"][0] + measurement["retrieved_ids"] = [] + errors = validate_report(curve_measurement, canonical=True) + assert any( + "fixed-budget curve 256 record recall_at_1" in error + and "retrieved_ids and supporting_ids" in error + for error in errors + ) + + +def test_canonical_validator_derives_numeric_grounded_metrics_from_labels(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + + unlabeled = _complete_canonical_report(dataset, config) + unlabeled["metrics"]["grounded_f1"] = 0.75 + unlabeled["metrics"]["abstention_f1"] = 0.75 + errors = validate_report(unlabeled, canonical=True) + assert any( + "metrics.grounded_f1 requires labeled per-question grounded values" in error + and "unavailable reason" in error + for error in errors + ) + assert any( + "metrics.abstention_f1 requires labeled per-question abstained values" in error + and "unavailable reason" in error + for error in errors + ) + + measured = _complete_canonical_report(dataset, config) + measured["records"][0].update({ + "answerable": True, + "grounded": True, + "abstained": False, + }) + measured["metrics"]["grounded"] = { + "available": True, + **metrics.grounded_precision_recall_f1([True], [True]), + } + measured["metrics"]["abstention"] = { + "available": True, + **metrics.abstention_precision_recall_f1([False], [True]), + } + measured["metrics"]["grounded_f1"] = 1.0 + measured["metrics"]["abstention_f1"] = 1.0 + assert validate_report(measured, canonical=True) == [] + + bad_count = deepcopy(measured) + bad_count["metrics"]["grounded"]["n"] = 2 + errors = validate_report(bad_count, canonical=True) + assert ( + "canonical metrics.grounded.n must be recomputed from per-question labels" + in errors + ) + + measured["metrics"]["grounded_f1"] = 0.0 + errors = validate_report(measured, canonical=True) + assert ( + "canonical metrics.grounded_f1 must be recomputed from per-question labels" + in errors + ) + + +def test_canonical_validator_requires_pinned_reader_tokenizer_identity(tmp_path): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + config = canonical_benchmark_config( + run_label="release-candidate", baseline_label="full_hybrid" + ) + valid = _complete_canonical_report(dataset, config) + + estimated = deepcopy(valid) + estimated["records"][0]["context_token_method"] = "deterministic_estimate" + estimated["metrics"]["fixed_budget_curve"]["rows"][0]["records"][0][ + "context_token_method" + ] = "deterministic_estimate" + errors = validate_report(estimated, canonical=True) + assert any( + "context_token_method=pinned_reader_content_tokenizer" in error + for error in errors + ) + assert any( + "fixed-budget curve 256 records require" in error + and "context_token_method=pinned_reader_content_tokenizer" in error + for error in errors + ) + + mismatched = deepcopy(valid) + mismatched["records"][0]["context_tokenizer_identity"] = "other/model@" + "e" * 40 + mismatched["records"][0]["usage"]["token_counter"] = "other/model@" + "e" * 40 + errors = validate_report(mismatched, canonical=True) + assert any("context_tokenizer_identity must match" in error for error in errors) + assert any("usage.token_counter must match" in error for error in errors) + + +def test_benchmark_cli_writes_canonical_json_and_checksum(tmp_path, capsys): + dataset = tmp_path / "fixture.jsonl" + dataset.write_text('{"id":"one"}\n', encoding="utf-8") + report = report_envelope( + suite="fixture", dataset_path=dataset, config={"k": 5}, + records=[question_record("q1")], git_commit="abc123", + ) + source = tmp_path / "source.json" + source.write_text(json.dumps(report), encoding="utf-8") + artifact = tmp_path / "artifact.json" + assert main(["--input", str(source), "--output", str(artifact)]) == 0 + assert artifact.exists() and artifact.with_name("artifact.json.sha256").exists() + assert "sha256" in capsys.readouterr().out + + +def test_exact_tokenizer_fallback_budget_curves_and_deterministic_cis(): + assert count_tokens("abc", CharacterTokenizer()) == {"tokens": 3, "method": "injected"} + assert count_tokens("one two")["method"] == "deterministic_estimate" + records = [ + {"category": "a", "supporting_ids": ["m1"], "chunks": [ + {"id": "m1", "tokens": 3}, {"id": "m2", "tokens": 3} + ]}, + {"category": "b", "supporting_ids": ["m2"], "chunks": [ + {"id": "m1", "tokens": 3}, {"id": "m2", "tokens": 3} + ]}, + ] + curve = fixed_budget_curve(records, [3, 6]) + assert curve[0]["recall"] == 0.5 + assert curve[1]["recall"] == 1.0 + def metric(rows): + return sum(row["value"] for row in rows) / len(rows) + ci_one = stratified_bootstrap_ci( + [{"category": "a", "value": 1.0}, {"category": "b", "value": 0.0}], + metric, iterations=40, seed=4, + ) + ci_two = stratified_bootstrap_ci( + [{"category": "a", "value": 1.0}, {"category": "b", "value": 0.0}], + metric, iterations=40, seed=4, + ) + assert ci_one == ci_two + paired = paired_bootstrap_ci([(1.0, 0.0), (0.0, 0.0)], iterations=40, seed=4) + assert paired["delta"] == 0.5 and paired["n"] == 2 diff --git a/tests/test_benchmark_longmemeval_v2.py b/tests/test_benchmark_longmemeval_v2.py new file mode 100644 index 00000000..2ef035cb --- /dev/null +++ b/tests/test_benchmark_longmemeval_v2.py @@ -0,0 +1,413 @@ +import json +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from eval.longmemeval_v2 import ( + EngraphisLongMemEvalV2Memory, + _context_items_with_budget, + _trajectory_text, +) +from eval import run_longmemeval_v2 +from eval.run_longmemeval_v2 import ( + PINNED_LONGMEMEVAL_V2_REVISION, + PINNED_READER_MODEL, + PINNED_READER_REVISION, + pin_official_reader_processor, + verify_official_checkout, +) + + +def test_v2_adapter_implements_official_insert_query_text_contract(): + memory = EngraphisLongMemEvalV2Memory(context_k=3) + memory.insert({ + "trajectory_id": "t-1", + "steps": [{"observation": "The billing page has an export button."}], + }) + context = memory.query("Where is the export button?") + assert context + assert all(item["type"] == "text" and item["value"] for item in context) + assert "export button" in context[0]["value"] + + +def test_v2_adapter_query_is_observational_and_writes_no_receipt(): + memory = EngraphisLongMemEvalV2Memory(context_k=3) + memory.insert({"id": "receipt-free", "text": "The billing page has an export button."}) + before_receipts = memory.service.store.conn.execute( + "SELECT COUNT(*) FROM operation_receipts" + ).fetchone()[0] + before_memory = memory.service.store.conn.execute( + "SELECT access_count, stability FROM memories" + ).fetchone() + + memory.query("Where is the export button?") + + after_receipts = memory.service.store.conn.execute( + "SELECT COUNT(*) FROM operation_receipts" + ).fetchone()[0] + after_memory = memory.service.store.conn.execute( + "SELECT access_count, stability FROM memories" + ).fetchone() + assert after_receipts == before_receipts + assert tuple(after_memory) == tuple(before_memory) + + +def test_v2_adapter_indexes_late_trajectory_states_without_full_history_duplication(): + memory = EngraphisLongMemEvalV2Memory(context_k=4, retrieval_profile="lexical") + memory.insert({ + "trajectory_id": "long", + "states": [ + {"observation": "Earlier state " + "noise " * 600}, + {"observation": "Late state: the recovery code is cobalt-owl-74."}, + ], + }) + + rows = memory.service.store.conn.execute( + "SELECT title, content, metadata FROM memories ORDER BY title" + ).fetchall() + assert len(rows) >= 2 + assert all(len(row["content"]) <= 1800 for row in rows) + assert all("state:" in row["title"] and "part:" in row["title"] for row in rows) + assert sum("cobalt-owl-74" in row["content"] for row in rows) == 1 + assert all("Earlier state" not in row["content"] or "Late state" not in row["content"] for row in rows) + + context = memory.query("What is the recovery code?") + assert any("cobalt-owl-74" in item["value"] for item in context) + + +def test_v2_adapter_flattens_common_trajectory_shapes_without_model_calls(): + assert _trajectory_text({"text": "direct record"}) == "direct record" + assert "action: click export" in _trajectory_text({"steps": [{"action": "click export"}]}) + + +def test_v2_adapter_flattens_official_states_and_legacy_content_shapes(): + state_text = _trajectory_text({ + "states": [{ + "accessibility_tree": "button Export", + "text": "Billing page", + "action": "click export", + "thought": "look for export", + "thoughts": ["confirm result"], + "url": "https://example.test/billing", + }], + }) + assert "accessibility_tree: button Export" in state_text + assert "thought: confirm result" in state_text + assert "url: https://example.test/billing" in state_text + + legacy_text = _trajectory_text({ + "content": [{ + "observation": {"text": "Export is in the Billing toolbar."}, + "action": "click toolbar export", + }], + }) + assert "observation.text: Export is in the Billing toolbar." in legacy_text + assert "action: click toolbar export" in legacy_text + + +def test_v2_adapter_enforces_injected_reader_token_budget_and_exposes_metadata(): + def character_tokens(text): + return len(text) + + memory = EngraphisLongMemEvalV2Memory( + context_k=3, + max_context_tokens=24, + tokenizer=character_tokens, + tokenizer_identity="test.characters.v1", + retrieval_profile="lexical", + ) + memory.insert({"id": "long", "text": "export button " * 30}) + context = memory.query("where is the export button?", query_image="ignored") + assert sum(character_tokens(item["value"]) for item in context) <= 24 + query_metadata = memory.post_query_hook( + query="where is the export button?", + query_image=None, + memory_context=context, + ) + assert query_metadata["returned_context_tokens"] <= 24 + assert query_metadata["returned_context_items"] == len(context) + assert query_metadata["tokenizer"] == "test.characters.v1" + assert "query" not in query_metadata + assert memory.metadata == { + "memory_type": "engraphis", + "context_k": 3, + "max_context_tokens": 24, + "budget_curve_status": "single_operating_point", + "required_budget_matrix": [256, 512, 1024, 2048, 4096], + "tokenizer": "test.characters.v1", + "token_budget_method": "deterministic_estimate", + "token_budget_scope": "per_context_item_content_excluding_prompt_framing", + "retrieval_profile": "lexical", + "embed_model": "deterministic", + "embed_revision": None, + "vector_backend": "numpy", + "response_mode": "compact", + } + + +def test_v2_adapter_returns_separate_context_items_for_official_prefix_truncation(): + items = _context_items_with_budget( + "[1] first\nalpha\n\n[2] second\nbeta", + budget=31, + count=len, + ) + + assert [item["type"] for item in items] == ["text", "text"] + assert items[0]["value"].startswith("[1]") + assert items[1]["value"].startswith("[2]") + assert sum(len(item["value"]) for item in items) <= 31 + + +def test_v2_adapter_requires_a_positive_context_budget(): + with pytest.raises(ValueError, match="max_context_tokens"): + EngraphisLongMemEvalV2Memory(max_context_tokens=0) + + +def test_v2_adapter_accepts_official_memory_params_and_persists_sqlite_state(tmp_path): + memory = EngraphisLongMemEvalV2Memory({ + "context_k": 2, + "max_context_tokens": 96, + "tokenizer_identity": "official.reader.v1", + "require_exact_reader_tokenizer": False, + "reader_tokenizer_model": None, + "reader_tokenizer_revision": None, + "retrieval_profile": "lexical", + }) + assert memory.memory_params == { + "context_k": 2, + "max_context_tokens": 96, + "tokenizer_identity": "official.reader.v1", + "require_exact_reader_tokenizer": False, + "reader_tokenizer_model": None, + "reader_tokenizer_revision": None, + "retrieval_profile": "lexical", + "embed_model": None, + "embed_revision": None, + "vector_backend": "numpy", + } + memory.insert({"id": "saved", "text": "The export button is in the billing toolbar."}) + memory.save_memory(tmp_path) + assert (tmp_path / "engraphis.sqlite").is_file() + saved_config = json.loads((tmp_path / "memory_config.json").read_text(encoding="utf-8")) + restored = EngraphisLongMemEvalV2Memory(saved_config["memory_params"]) + restored._load_backend(tmp_path) + assert "billing toolbar" in restored.query("where is the export button?")[0]["value"] + + +def test_checked_in_official_memory_config_pins_embedding_backend(monkeypatch): + config_path = Path(__file__).resolve().parents[1] / "eval" / "configs" / "longmemeval_v2_engraphis.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + assert config["memory_type"] == "engraphis" + created = {} + + def create_service(path, **kwargs): + created.update({"path": path, **kwargs}) + return SimpleNamespace(engine=SimpleNamespace(embedder=SimpleNamespace( + model_name=kwargs["embed_model"], + revision=kwargs["embed_revision"], + ))) + + class ExactTokenizer: + def encode(self, text): + return list(text) + + tokenizer_request = {} + + def load_tokenizer(model, revision): + tokenizer_request.update({"model": model, "revision": revision}) + return ExactTokenizer() + + monkeypatch.setattr("eval.longmemeval_v2.MemoryService.create", create_service) + monkeypatch.setattr("eval.longmemeval_v2._load_pinned_reader_tokenizer", load_tokenizer) + memory = EngraphisLongMemEvalV2Memory(config["memory_params"]) + assert memory.max_context_tokens == 1024 + assert memory.retrieval_profile == "balanced" + assert config["memory_params"]["require_exact_reader_tokenizer"] is True + assert config["memory_params"]["reader_tokenizer_model"] == "Qwen/Qwen3.5-9B" + assert config["memory_params"]["reader_tokenizer_revision"] == ( + "c202236235762e1c871ad0ccb60c8ee5ba337b9a" + ) + assert config["memory_params"]["tokenizer_identity"] != "engraphis.regex.v1" + assert memory.embed_model == "Qwen/Qwen3-Embedding-8B" + assert memory.embed_revision == "1d8ad4ca9b3dd8059ad90a75d4983776a23d44af" + assert created["embed_model"] == memory.embed_model + assert created["embed_revision"] == memory.embed_revision + assert created["vector_backend"] == "numpy" + assert memory.require_exact_reader_tokenizer is True + assert memory.metadata["token_budget_method"] == "pinned_reader_content_tokenizer" + assert memory.metadata["budget_curve_status"] == "single_operating_point" + assert tokenizer_request == { + "model": "Qwen/Qwen3.5-9B", + "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + } + + +def test_v2_adapter_rejects_silent_deterministic_embedding_fallback(monkeypatch): + params = { + "embed_model": "Qwen/Qwen3-Embedding-8B", + "embed_revision": "1d8ad4ca9b3dd8059ad90a75d4983776a23d44af", + } + fallback = SimpleNamespace( + engine=SimpleNamespace(embedder=SimpleNamespace()) + ) + monkeypatch.setattr( + "eval.longmemeval_v2.MemoryService.create", + lambda *args, **kwargs: fallback, + ) + + with pytest.raises(RuntimeError, match="canonical fallback is forbidden"): + EngraphisLongMemEvalV2Memory(params) + + +def test_v2_adapter_fails_closed_when_canonical_reader_tokenizer_is_not_verified(monkeypatch): + params = { + "require_exact_reader_tokenizer": True, + "reader_tokenizer_model": "Qwen/Qwen3.5-9B", + "reader_tokenizer_revision": "a" * 40, + "tokenizer_identity": "Qwen/Qwen3.5-9B@" + "a" * 40, + } + + def unavailable(model, revision): + raise ValueError("reader tokenizer unavailable") + + monkeypatch.setattr("eval.longmemeval_v2._load_pinned_reader_tokenizer", unavailable) + with pytest.raises(ValueError, match="reader tokenizer unavailable"): + EngraphisLongMemEvalV2Memory(params) + + with pytest.raises(ValueError, match="requires reader_tokenizer_model"): + EngraphisLongMemEvalV2Memory({"require_exact_reader_tokenizer": True}) + with pytest.raises(ValueError, match="requires the pinned reader tokenizer identity"): + EngraphisLongMemEvalV2Memory( + params, + tokenizer_identity="engraphis.regex.v1", + ) + with pytest.raises(ValueError, match="does not accept an injected replacement"): + EngraphisLongMemEvalV2Memory( + params, + tokenizer=lambda text: len(text), + ) + + +def test_v2_adapter_requires_immutable_embedding_revision(): + with pytest.raises(ValueError, match="immutable"): + EngraphisLongMemEvalV2Memory( + embed_model="Qwen/Qwen3-Embedding-8B", + embed_revision="main", + ) + + +def test_v2_adapter_registers_as_an_official_memory_module_when_available(tmp_path): + package = tmp_path / "memory_modules" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "memory.py").write_text( + "MEMORY_TYPES = {}\n" + "class Memory:\n" + " def __init__(self, memory_params): self.memory_params = dict(memory_params)\n" + " def configure_runtime(self, **kwargs): pass\n" + "def register_memory(cls):\n" + " MEMORY_TYPES[cls.memory_type] = cls\n" + " return cls\n", + encoding="utf-8", + ) + root = Path(__file__).resolve().parents[1] + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join([str(tmp_path), str(root)]) + program = "\n".join([ + "import memory_modules.memory as official", + "import eval.longmemeval_v2 as adapter", + "assert adapter.OFFICIAL_MEMORY_AVAILABLE is True", + "memory_cls = official.MEMORY_TYPES['engraphis']", + "assert issubclass(memory_cls, official.Memory)", + "memory = memory_cls({'context_k': 3, 'max_context_tokens': 64})", + "assert memory.memory_params['context_k'] == 3", + ]) + subprocess.run([sys.executable, "-c", program], cwd=root, env=env, check=True) + + +def test_official_runner_requires_the_exact_pinned_checkout(monkeypatch, tmp_path): + module = SimpleNamespace(__file__=str(tmp_path / "memory.py")) + commands = [] + + def pinned_check_output(command, **kwargs): + commands.append(command) + if command[-1] == "--show-toplevel": + return str(tmp_path) + "\n" + return PINNED_LONGMEMEVAL_V2_REVISION + "\n" + + monkeypatch.setattr("eval.run_longmemeval_v2.subprocess.check_output", pinned_check_output) + verify_official_checkout(module) + assert commands + + def mismatched_check_output(command, **kwargs): + if command[-1] == "--show-toplevel": + return str(tmp_path) + "\n" + return "a" * 40 + "\n" + + monkeypatch.setattr("eval.run_longmemeval_v2.subprocess.check_output", mismatched_check_output) + with pytest.raises(SystemExit, match="revision mismatch"): + verify_official_checkout(module) + + +def test_official_runner_verifies_checkout_before_registering_or_delegating(monkeypatch): + memory_module = SimpleNamespace(__file__="C:/official/memory_modules/memory.py") + calls = [] + + def import_module(name): + calls.append(("import", name)) + return memory_module if name == "memory_modules.memory" else object() + + monkeypatch.setattr(run_longmemeval_v2.importlib, "import_module", import_module) + monkeypatch.setattr( + run_longmemeval_v2, + "verify_official_checkout", + lambda module: calls.append(("verify", module)), + ) + monkeypatch.setattr( + run_longmemeval_v2.runpy, + "run_module", + lambda name, run_name: calls.append(("run", name, run_name)), + ) + monkeypatch.setattr( + run_longmemeval_v2, + "pin_official_reader_processor", + lambda: (lambda: calls.append(("restore_reader",))), + ) + + run_longmemeval_v2.main() + + assert calls == [ + ("import", "memory_modules.memory"), + ("verify", memory_module), + ("import", "eval.longmemeval_v2"), + ("run", "evaluation.harness", "__main__"), + ("restore_reader",), + ] + + +def test_official_runner_forces_the_reader_processor_to_the_pinned_revision(monkeypatch): + requests = [] + + class FakeProcessor: + @classmethod + def from_pretrained(cls, model, *args, **kwargs): + requests.append((model, args, kwargs)) + return object() + + monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(AutoProcessor=FakeProcessor)) + restore = pin_official_reader_processor() + try: + FakeProcessor.from_pretrained(PINNED_READER_MODEL) + assert requests == [ + (PINNED_READER_MODEL, (), {"revision": PINNED_READER_REVISION}) + ] + with pytest.raises(RuntimeError, match="outside the canonical profile"): + FakeProcessor.from_pretrained(PINNED_READER_MODEL, revision="a" * 40) + with pytest.raises(RuntimeError, match="only the configured reader processor"): + FakeProcessor.from_pretrained("different-reader") + finally: + restore() diff --git a/tests/test_eval_external.py b/tests/test_eval_external.py index 81d77e3b..d46c517f 100644 --- a/tests/test_eval_external.py +++ b/tests/test_eval_external.py @@ -1,6 +1,8 @@ import json -from eval.external import load_locomo, load_longmemeval +import pytest + +from eval.external import load_locomo, load_longmemeval, main, source_case_count from eval.harness import run @@ -68,21 +70,35 @@ def test_load_locomo_normalizes_to_harness_cases(tmp_path): tags = {m["tag"] for m in case["memories"]} assert tags == {"D1:1", "D1:2", "D2:1"} assert case["memories"][0]["text"].startswith("[1:00 pm on 8 May, 2023] Caroline:") - assert len(case["questions"]) == 1 # adversarial (no evidence) skipped + assert len(case["questions"]) == 2 # adversarial is retained and explicit assert case["questions"][0]["supporting"] == ["D1:1"] + assert case["questions"][1]["category"] == "5" + assert case["questions"][1]["answerable"] is False def test_load_longmemeval_sessions_and_abstention(tmp_path): cases = load_longmemeval(_longmemeval_fixture(tmp_path)) - assert len(cases) == 1 # _abs instance skipped + assert len(cases) == 2 # _abs instance is retained case = cases[0] assert {m["tag"] for m in case["memories"]} == {"s1", "s2"} assert "pnpm" in case["memories"][0]["text"] assert case["questions"][0]["supporting"] == ["s1"] + assert cases[1]["questions"][0]["category"] == "abstention" + assert cases[1]["questions"][0]["answerable"] is False def test_external_cases_run_through_the_real_harness(tmp_path): cases = load_locomo(_locomo_fixture(tmp_path)) report = run(cases, k=3) # offline deterministic embedder - assert report["questions"] == 1 + assert report["questions"] == 2 + assert report["scored_questions"] == 1 + assert report["exclusions"][0]["reason"] == "no_gold_evidence" assert report["recall_at_k"] == 1.0 # evidence found in a 3-memory haystack + + +def test_canonical_external_mode_rejects_partial_limit_before_model_loading(tmp_path): + path = _locomo_fixture(tmp_path) + assert source_case_count(path) == 1 + with pytest.raises(SystemExit) as error: + main(["--dataset", path, "--format", "locomo", "--canonical", "--limit", "1"]) + assert error.value.code == 2 diff --git a/tests/test_eval_harness.py b/tests/test_eval_harness.py index 32308913..c210005e 100644 --- a/tests/test_eval_harness.py +++ b/tests/test_eval_harness.py @@ -1,8 +1,33 @@ from pathlib import Path -from eval.harness import load_dataset, run +import pytest + +from eval.benchmark import LONGMEMEVAL_V2_CANONICAL_PROFILE_TEMPLATE, validate_report +from engraphis.backends import DeterministicEmbedder +from engraphis.core.interfaces import SearchFilter +from engraphis.core.store import Store +from eval.harness import ( + _seed_case_graph, + executable_baseline, + load_dataset, + main as harness_main, + paired_v2_bootstrap, + run, + run_baseline_matrix, +) DATASET = Path(__file__).resolve().parent.parent / "eval" / "datasets" / "sample.jsonl" +GRAPH_DATASET = ( + Path(__file__).resolve().parent.parent / "eval" / "datasets" / "graph_multihop.jsonl" +) + + +class FakePinnedReaderCounter: + def __init__(self, model, revision): + self.identity = f"{model}@{revision}" + + def __call__(self, text): + return len(text.split()) def test_harness_runs_and_scores(): @@ -14,6 +39,25 @@ def test_harness_runs_and_scores(): assert report["recall_at_k"] > 0.5 +def test_harness_seeds_declared_fixture_graph_before_memory_ingestion(): + case = load_dataset(str(GRAPH_DATASET))[0] + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("eval") + repo_id = store.get_or_create_repo(workspace_id, case["id"]) + + _seed_case_graph( + store, + workspace_id=workspace_id, + repo_id=repo_id, + case=case, + ) + + flt = SearchFilter(workspace_id=workspace_id, repo_id=repo_id) + assert len(store.list_entities(flt)) == 3 + assert len(store.edges_in_scope(flt)) == 2 + store.close() + + def test_metrics_edges(): from eval import metrics assert metrics.recall_at_k([], []) == 1.0 @@ -21,3 +65,350 @@ def test_metrics_edges(): assert metrics.hit_at_k(["a"], ["b"]) == 0.0 assert metrics.answer_token_recall(["redis lock around stock decrement"], "Redis lock") == 1.0 + + +def test_grounded_and_abstention_binary_metrics_are_explicit(): + from eval import metrics + + grounded = metrics.grounded_precision_recall_f1( + [True, False, True], [True, True, False] + ) + assert grounded == { + "precision": 0.5, "recall": 0.5, "f1": 0.5, + "true_positive": 1, "false_positive": 1, "false_negative": 1, "n": 3, + } + abstention = metrics.abstention_precision_recall_f1( + [False, True, True], [True, True, False] + ) + assert abstention["precision"] == 0.5 + assert abstention["recall"] == 1.0 + assert abstention["f1"] == pytest.approx(2 / 3) + with pytest.raises(ValueError, match="equal length"): + metrics.binary_precision_recall_f1([True], []) + + +def test_v2_harness_envelope_records_usage_latency_and_rank_metrics(): + report = run(load_dataset(str(DATASET)), k=3, v2=True, dataset_path=str(DATASET), + bootstrap_iterations=8) + assert report["schema"] == "engraphis-benchmark/v2" + assert len(report["suite"]["sha256"]) == 64 + assert len(report["system"]["config_sha256"]) == 64 + assert len(report["models"]["embedder"]["sha256"]) == 64 + first = report["records"][0] + assert {"usage", "latency_ms", "recall_at_1", "recall_at_5", "recall_at_10", + "mrr_at_5", "ndcg_at_10"} <= set(first) + assert {"budget_tokens", "context_tokens", "source_tokens", "saved_tokens", + "savings_ratio", "packed_count", "omitted_count", "token_counter"} <= set(first["usage"]) + metrics = report["metrics"] + assert {"recall_at_1", "recall_at_5", "recall_at_10", "mrr_at_5", "ndcg_at_5", + "confidence_intervals", "paired_bootstrap"} <= set(metrics) + assert metrics["confidence_intervals"]["recall_at_5"]["iterations"] == 8 + assert metrics["paired_bootstrap"]["available"] is False + assert report["legacy_summary"]["questions"] == 4 + + +def test_canonical_harness_requires_pinned_profile_and_complete_artifact(monkeypatch): + with pytest.raises(ValueError, match="pinned revisions"): + run(load_dataset(str(DATASET)), canonical=True, dataset_path=str(DATASET)) + + profile = { + **LONGMEMEVAL_V2_CANONICAL_PROFILE_TEMPLATE, + "benchmark": { + "repository": "example/benchmark", "repository_revision": "a" * 40, + "dataset_revision": "b" * 40, + }, + "reader": {"model": "example/reader", "revision": "c" * 40}, + "embedding": {"model": "example/embedder", "revision": "d" * 40}, + "baseline_label": "full_hybrid", + } + monkeypatch.setattr( + "eval.harness._load_pinned_reader_token_counter", + lambda model, revision: FakePinnedReaderCounter(model, revision), + ) + canonical_embedder = DeterministicEmbedder(dim=256) + canonical_embedder.model_name = "example/embedder" + canonical_embedder.revision = "d" * 40 + with pytest.raises(ValueError, match="positive bootstrap_iterations"): + run( + load_dataset(str(DATASET)), + canonical=True, + embedder=canonical_embedder, + dataset_path=str(DATASET), + canonical_profile=profile, + bootstrap_iterations=0, + ) + report = run(load_dataset(str(DATASET)), k=3, canonical=True, embedder=canonical_embedder, + dataset_path=str(DATASET), canonical_profile=profile, + bootstrap_iterations=2) + assert report["protocol"]["complete_dataset"] is True + assert report["protocol"]["source_questions"] == len(report["records"]) + assert validate_report(report, canonical=True) == [] + curve = report["metrics"]["fixed_budget_curve"] + assert curve["available"] is True + assert [row["token_budget"] for row in curve["rows"]] == [256, 512, 1024, 2048, 4096] + assert all(row["status"] == "measured" for row in curve["rows"]) + assert all(len(row["records"]) == len(report["records"]) for row in curve["rows"]) + assert len(report["models"]["embedder"]["sha256"]) == 64 + assert "q" not in report["records"][0] + assert len(report["records"][0]["question_sha256"]) == 64 + assert ( + report["records"][0]["context_token_method"] + == "pinned_reader_content_tokenizer" + ) + assert report["records"][0]["context_tokenizer_identity"] == ( + "example/reader@" + "c" * 40 + ) + assert report["records"][0]["usage"]["token_counter"] == ( + "example/reader@" + "c" * 40 + ) + with pytest.raises(ValueError, match="model_name and revision"): + run(load_dataset(str(DATASET)), k=3, canonical=True, + dataset_path=str(DATASET), canonical_profile=profile, + bootstrap_iterations=2) + mismatch = {**profile, "baseline_label": "lexical_only"} + with pytest.raises(ValueError, match="must match the executed baseline_label"): + run(load_dataset(str(DATASET)), canonical=True, dataset_path=str(DATASET), + canonical_profile=mismatch, bootstrap_iterations=2) + + +def test_canonical_budget_curve_scores_only_packed_evidence(monkeypatch): + import eval.harness as harness + from engraphis.core.interfaces import ContextUsage, PackedChunk + from engraphis.core.recall import RecallResult + + def controlled_recall( + engine, query, *, workspace_id, repo_id, k, token_budget, baseline, + source_records=None, + ): + del query, k, baseline, source_records + record = engine.store.list_memories( + SearchFilter(workspace_id=workspace_id, repo_id=repo_id) + )[0] + budget = 1500 if token_budget is None else token_budget + admitted = budget != 256 + packed = ( + [PackedChunk(id=record.id, excerpt=record.content, tokens=1, reason="test")] + if admitted else [] + ) + return RecallResult( + chunks=[{"id": record.id, "title": record.title, "content": record.content}], + packed_chunks=packed, + context=record.content if admitted else "", + count=1, + usage=ContextUsage( + budget_tokens=budget, + context_tokens=1 if admitted else 0, + source_tokens=1, + saved_tokens=0 if admitted else 1, + savings_ratio=0.0 if admitted else 1.0, + packed_count=1 if admitted else 0, + omitted_count=0 if admitted else 1, + token_counter="test", + ), + ) + + monkeypatch.setattr(harness, "_recall_for_baseline", controlled_recall) + monkeypatch.setattr( + harness, + "_load_pinned_reader_token_counter", + lambda model, revision: FakePinnedReaderCounter(model, revision), + ) + profile = { + **LONGMEMEVAL_V2_CANONICAL_PROFILE_TEMPLATE, + "benchmark": { + "repository": "example/benchmark", "repository_revision": "a" * 40, + "dataset_revision": "b" * 40, + }, + "reader": {"model": "example/reader", "revision": "c" * 40}, + "embedding": {"model": "example/embedder", "revision": "d" * 40}, + "baseline_label": "full_hybrid", + } + embedder = DeterministicEmbedder(dim=256) + embedder.model_name = "example/embedder" + embedder.revision = "d" * 40 + dataset = [{ + "id": "packed-budget", + "memories": [{"tag": "gold", "text": "The release train leaves Tuesday."}], + "questions": [{ + "id": "q1", + "q": "when does the release train leave", + "supporting": ["gold"], + }], + }] + + report = run( + dataset, + canonical=True, + dataset_path=str(DATASET), + canonical_profile=profile, + embedder=embedder, + bootstrap_iterations=2, + ) + rows = { + row["token_budget"]: row + for row in report["metrics"]["fixed_budget_curve"]["rows"] + } + assert rows[256]["recall_at_1"] == 0.0 + assert rows[256]["records"][0]["context_tokens"] == 0 + assert rows[512]["recall_at_1"] == 1.0 + assert rows[512]["records"][0]["context_tokens"] == 1 + + +def test_paired_bootstrap_rejects_partial_comparisons(): + candidate = [{"question_id": "a", "recall_at_5": 1.0}, + {"question_id": "b", "recall_at_5": 0.0}] + baseline = [{"question_id": "a", "recall_at_5": 0.0}, + {"question_id": "b", "recall_at_5": 0.0}] + paired = paired_v2_bootstrap(candidate, baseline, iterations=8) + assert paired["available"] is True and paired["delta"] == 0.5 + with pytest.raises(ValueError, match="identical scored question IDs"): + paired_v2_bootstrap(candidate, baseline[:1]) + + +def test_harness_cli_keeps_legacy_default_and_offers_opt_in_v2_artifacts(tmp_path, capsys): + artifact = tmp_path / "run.json" + harness_main([ + "--dataset", str(DATASET), "--v2", "--artifact", str(artifact), + "--bootstrap-iterations", "2", + ]) + assert artifact.exists() and artifact.with_name("run.json.sha256").exists() + assert '"schema": "engraphis-benchmark/v2"' in capsys.readouterr().out + with pytest.raises(SystemExit, match="2"): + harness_main(["--dataset", str(DATASET), "--canonical"]) + + +def test_harness_baselines_execute_declared_retrieval_arms(monkeypatch): + from engraphis.core.recall import RecallEngine + + seen = [] + original = RecallEngine.recall + + def observe(self, *args, **kwargs): + seen.append(kwargs.get("arm_config")) + return original(self, *args, **kwargs) + + monkeypatch.setattr(RecallEngine, "recall", observe) + data = load_dataset(str(DATASET)) + for label in ("full_hybrid", "dense_only", "lexical_only", "dense_lexical_rrf", "no_graph"): + run(data, k=2, baseline_label=label) + config = seen[-1] + expected = executable_baseline(label) + assert (config.vector, config.lexical, config.graph, config.code) == ( + expected.vector, expected.lexical, expected.graph, False, + ) + + no_retrieval = run(data, k=2, baseline_label="no_retrieval") + assert no_retrieval["baseline_execution"]["no_retrieval"] is True + assert all(not item["retrieved_ids"] for item in no_retrieval["detail"]) + + +def test_harness_baseline_matrix_and_canonical_labels_fail_closed(): + data = load_dataset(str(DATASET)) + matrix = run_baseline_matrix( + data, baseline_labels=("dense_only", "lexical_only", "no_graph", "full_hybrid"), k=2, + ) + assert set(matrix) == {"dense_only", "lexical_only", "no_graph", "full_hybrid"} + assert matrix["dense_only"]["baseline_execution"]["arms"] == { + "vector": True, "lexical": False, "graph": False, "code": False, + } + assert matrix["lexical_only"]["baseline_execution"]["arms"] == { + "vector": False, "lexical": True, "graph": False, "code": False, + } + assert matrix["no_graph"]["baseline_execution"]["arms"] == { + "vector": True, "lexical": True, "graph": False, "code": False, + } + rrf = run(data, baseline_label="dense_lexical_rrf") + assert rrf["baseline_execution"]["equivalent_to"] == "no_graph" + with pytest.raises(ValueError, match="requires a non-empty document"): + run(data, baseline_label="whole_document") + with pytest.raises(ValueError, match="requires ordered non-empty memories"): + run_baseline_matrix( + [{"id": "document-only", "document": "source", "questions": []}], + baseline_labels=("full_history",), + ) + + +def test_harness_executes_corpus_and_temporal_baselines_only_when_representable(): + data = load_dataset(str(DATASET)) + history = run(data, baseline_label="full_history") + assert history["baseline_execution"]["mode"] == "full_history" + assert history["detail"][0]["usage"]["saved_tokens"] == 0 + assert set(history["detail"][0]["retrieved_ids"]) == {"f1", "f2", "f3", "f4"} + + document = [{ + "id": "doc", "document": "The billing export is in the settings menu.", + "questions": [{"q": "where is the billing export", "evidence": "settings menu"}], + }] + whole = run(document, baseline_label="whole_document") + assert whole["baseline_execution"]["mode"] == "whole_document" + assert whole["detail"][0]["retrieved_ids"] == ["whole_document"] + assert whole["detail"][0]["usage"]["saved_tokens"] == 0 + + whole_with_sources = run([{ + "id": "document-with-sources", + "document": "The billing export is in the settings menu.", + "memories": [{"tag": "billing-settings", "text": "The billing export is in settings."}], + "questions": [{ + "q": "where is the billing export", "supporting": ["billing-settings"], + }], + }], baseline_label="whole_document") + assert whole_with_sources["detail"][0]["retrieved_ids"] == ["billing-settings"] + assert whole_with_sources["detail"][0]["recall_at_k"] == 1.0 + + temporal = [{ + "id": "temporal", + "memories": [ + {"tag": "old", "text": "The plan price is ten dollars.", "subject_key": "plan-price", + "claim_kind": "price", "valid_from": 1.0}, + {"tag": "new", "text": "The plan price is twenty dollars.", "subject_key": "plan-price", + "claim_kind": "price", "valid_from": 2.0}, + ], + "questions": [{"q": "what is the plan price", "supporting": ["new"]}], + }] + temporal_report = run(temporal, k=5, baseline_label="no_temporal_resolution") + assert temporal_report["baseline_execution"]["temporal_resolution"] == "disabled" + assert {"old", "new"} <= set(temporal_report["detail"][0]["retrieved_ids"]) + + +def test_harness_no_reranker_requires_and_disables_a_real_reranker(): + class ReverseReranker: + def __init__(self): + self.calls = 0 + + def rerank(self, query, candidates, k): + self.calls += 1 + return list(reversed(candidates))[:k] + + with pytest.raises(ValueError, match="non-identity reranker"): + run(load_dataset(str(DATASET)), baseline_label="no_reranker") + reranker = ReverseReranker() + report = run(load_dataset(str(DATASET)), baseline_label="no_reranker", reranker=reranker) + assert reranker.calls == 0 + assert report["baseline_execution"]["reranker"] == "disabled" + + +def test_harness_v2_grounded_and_abstention_metrics_are_available_or_explicitly_unavailable(): + data = [{ + "id": "grounded", + "memories": [{"tag": "fact", "text": "The release train leaves on Tuesday."}], + "questions": [ + {"id": "answerable", "q": "when does the release train leave", "supporting": ["fact"], + "answerable": True}, + {"id": "unanswerable", "q": "what is the moon made of", "supporting": [], + "answerable": False}, + ], + }] + unavailable = run(data, v2=True, dataset_path=str(DATASET), bootstrap_iterations=2) + assert unavailable["metrics"]["grounded"] == { + "available": False, "reason": "grounded_recall_not_run", "n": 2, + } + assert unavailable["metrics"]["grounded_f1"] == { + "available": False, "reason": "grounded_recall_not_run", "n": 2, + } + assert unavailable["protocol"]["n_scored"] == 1 + + available = run(data, v2=True, dataset_path=str(DATASET), bootstrap_iterations=2, grounded=True) + assert available["metrics"]["grounded"]["available"] is True + assert available["metrics"]["abstention"]["available"] is True + assert available["metrics"]["grounded"]["n"] == 2 + assert available["metrics"]["abstention"]["n"] == 2 diff --git a/tests/test_eval_performance.py b/tests/test_eval_performance.py new file mode 100644 index 00000000..ead27cd5 --- /dev/null +++ b/tests/test_eval_performance.py @@ -0,0 +1,211 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from engraphis.core.interfaces import ContextUsage, PackedChunk +from eval import performance +from eval.performance import AcceptanceConfig, main, run + + +DATASET = [ + { + "id": "auth", + "memories": [ + {"tag": "token", "text": "The API authenticates with PASETO v4 tokens."}, + {"tag": "deploy", "text": "Deployments run through GitHub Actions."}, + ], + "questions": [ + { + "q": "Which token format authenticates the API?", + "answer": "PASETO v4", + "supporting": ["token"], + } + ], + } +] + + +def test_performance_report_covers_quality_context_and_latency(): + report = run(DATASET, k=2, warmups=0, iterations=2, filler_memories=3) + + assert report["schema"] == "engraphis-performance/v1" + assert report["corpus"] == { + "dataset_cases": 1, + "memories": 5, + "questions": 1, + "filler_memories": 3, + } + assert report["run"]["timed_recalls"] == 2 + assert report["quality"]["hit_at_k"] == 1.0 + assert report["context"]["mean_tokens"] > 0 + assert report["context"]["token_counter"] == "engraphis.regex.v1" + assert -1 <= report["context"]["median_serialized_payload_savings_ratio"] <= 1 + assert report["context"]["saved_serialized_payload_tokens"] == ( + report["context"]["full_serialized_payload_tokens"] + - report["context"]["compact_serialized_payload_tokens"] + ) + assert 0 <= report["latency_ms"]["min"] <= report["latency_ms"]["p50"] + assert report["latency_ms"]["p50"] <= report["latency_ms"]["p95"] + assert report["latency_ms"]["p95"] <= report["latency_ms"]["max"] + + +def test_codemem_median_compact_payload_savings_clears_release_gate(): + dataset = performance.load_dataset( + Path(__file__).resolve().parents[1] / "eval" / "datasets" / "codemem.jsonl" + ) + + report = run(dataset, k=5, warmups=0, iterations=1) + + assert report["context"]["median_serialized_payload_savings_ratio"] >= 0.5 + + +def test_performance_report_separates_cold_warm_and_acceptance_shape(): + report = run(DATASET, k=2, warmups=0, iterations=1, concurrency=4) + + assert report["acceptance"] == { + "concurrency": 4, + "independent_processes": 1, + "minimum_queries": 0, + "canonical": False, + "query_count": 1, + "valid": True, + } + assert report["run"]["cold_timed_recalls"] == 1 + assert report["run"]["warm_timed_recalls"] == 1 + assert report["latency_ms"]["cold"]["p50"] >= 0 + assert report["latency_ms"]["warm"]["p99"] >= report["latency_ms"]["warm"]["p50"] + assert report["resources"]["processes"][0]["storage_bytes"] is not None + + +def test_compact_payload_mirrors_packed_mcp_sources_in_ordinal_order(): + result = SimpleNamespace( + context="[1] second\n[2] first", + chunks=[ + {"id": "mem_first", "title": "First", "scope": "repo", "score": 0.5}, + { + "id": "mem_second", + "title": "Second", + "scope": "repo", + "score": 0.9, + "provenance": {"source": "agent", "secret": "not forwarded"}, + }, + {"id": "mem_unpacked", "title": "Unpacked", "scope": "repo", "score": 0.1}, + ], + packed_chunks=[ + PackedChunk("mem_second", "second", 3, False, "full"), + PackedChunk("mem_first", "first", 2, True, "summary"), + ], + usage=ContextUsage(20, 12, 30, 18, 0.6, 2, 1), + ) + + compact = performance._compact_payload(result) + + assert [source["id"] for source in compact["sources"]] == ["mem_second", "mem_first"] + assert [source["n"] for source in compact["sources"]] == [1, 2] + assert all(source["id"] != "mem_unpacked" for source in compact["sources"]) + assert compact["sources"][1]["truncated"] is True + assert "reason" not in compact["sources"][1] + assert compact["sources"][0]["provenance"] == {"source": "agent"} + assert set(compact["sources"][0]) == {"n", "id", "tokens", "title", "provenance"} + assert compact["usage"]["context_tokens"] == 12 + + +@pytest.mark.parametrize( + ("config", "question_count", "message"), + [ + (AcceptanceConfig(concurrency=2), 1, "concurrency"), + (AcceptanceConfig(processes=0), 1, "processes"), + (AcceptanceConfig(minimum_queries=2), 1, "minimum_queries"), + (AcceptanceConfig(canonical=True, processes=5), 999, "1000 queries"), + (AcceptanceConfig(canonical=True, processes=4), 1000, "5 processes"), + ], +) +def test_acceptance_config_rejects_invalid_protocols( + config: AcceptanceConfig, question_count: int, message: str +): + with pytest.raises(ValueError, match=message): + config.validate(question_count) + + +def test_canonical_acceptance_validation_does_not_require_a_large_run(): + AcceptanceConfig(concurrency=16, processes=5, minimum_queries=1000, canonical=True).validate( + 1000 + ) + + +def test_per_slice_run_rejects_a_canonical_claim(): + with pytest.raises(ValueError, match="run_acceptance_matrix"): + run(DATASET, canonical=True) + + +def test_acceptance_matrix_requires_every_declared_concurrency(): + with pytest.raises(ValueError, match="every concurrency"): + performance.run_acceptance_matrix([], concurrencies=[1, 4], processes=5) + + +def test_acceptance_matrix_groups_each_slice_without_running_a_large_benchmark(monkeypatch): + calls = [] + + def fake_run(dataset, **kwargs): + calls.append(kwargs) + return {"schema": "engraphis-performance/v1", "acceptance": kwargs} + + monkeypatch.setattr(performance, "_question_count", lambda dataset: 1000) + monkeypatch.setattr(performance, "run", fake_run) + + report = performance.run_acceptance_matrix([], processes=5) + + assert [call["concurrency"] for call in calls] == [1, 4, 16] + assert all(call["canonical"] is False for call in calls) + assert all(call["processes"] == 5 for call in calls) + assert report["acceptance"]["concurrency_matrix"] == [1, 4, 16] + assert list(report["slices"]) == ["1", "4", "16"] + + +def test_cli_acceptance_matrix_uses_matrix_runner(tmp_path, capsys, monkeypatch): + dataset = tmp_path / "cases.jsonl" + dataset.write_text(json.dumps(DATASET[0]) + "\n", encoding="utf-8") + expected = { + "schema": "engraphis-performance-matrix/v1", + "acceptance": {"valid": True}, + "slices": {}, + } + calls = [] + + def fake_matrix(dataset, **kwargs): + calls.append(kwargs) + return expected + + monkeypatch.setattr(performance, "run_acceptance_matrix", fake_matrix) + + assert main(["--dataset", str(dataset), "--acceptance-matrix", "--json"]) == 0 + + assert calls == [{ + "k": 5, + "dim": 256, + "warmups": 1, + "iterations": 5, + "filler_memories": 0, + "token_budget": 1500, + "processes": 1, + "minimum_queries": 0, + }] + assert json.loads(capsys.readouterr().out) == expected + + +def test_performance_cli_emits_json(tmp_path, capsys): + dataset = tmp_path / "cases.jsonl" + dataset.write_text(json.dumps(DATASET[0]) + "\n", encoding="utf-8") + + assert main([ + "--dataset", str(dataset), + "--iterations", "1", + "--warmups", "0", + "--json", + ]) == 0 + + report = json.loads(capsys.readouterr().out) + assert report["corpus"]["questions"] == 1 + assert report["latency_ms"]["p95"] >= 0 diff --git a/tests/test_release_evidence.py b/tests/test_release_evidence.py new file mode 100644 index 00000000..5a698d59 --- /dev/null +++ b/tests/test_release_evidence.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from engraphis.service import MemoryService +from scripts.release_evidence import ( + EvidenceError, + build_evidence, + canonical_json_bytes, + check_manifest, +) + + +COMMIT = "a" * 40 +TAG = "v1.2.3" +ROOT = Path(__file__).resolve().parents[1] + + +def _root(tmp_path): + (tmp_path / "eval" / "datasets").mkdir(parents=True) + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "engraphis"\nversion = "1.2.3"\n', encoding="utf-8" + ) + (tmp_path / "LICENSE").write_text("Apache-2.0\n", encoding="utf-8") + (tmp_path / "NOTICE").write_text("Engraphis\n", encoding="utf-8") + (tmp_path / "eval" / "datasets" / "sample.jsonl").write_text('{"id":"sample"}\n') + (tmp_path / "eval" / "datasets" / "codemem.jsonl").write_text('{"id":"code"}\n') + (tmp_path / "eval" / "datasets" / "graph_multihop.jsonl").write_text( + '{"id":"graph"}\n', encoding="utf-8" + ) + return tmp_path + + +def _dist(root): + directory = root / "dist" + directory.mkdir() + (directory / "engraphis-1.2.3-py3-none-any.whl").write_bytes(b"wheel") + (directory / "engraphis-1.2.3.tar.gz").write_bytes(b"sdist") + return directory + + +def _sbom(root): + path = root / "release-evidence" / "engraphis-1.2.3.cdx.json" + path.parent.mkdir(exist_ok=True) + path.write_text( + json.dumps( + { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [{"type": "library", "name": "engraphis", "version": "1.2.3"}], + } + ), + encoding="utf-8", + ) + return path + + +def _check_ids(root): + return [entry["id"] for group in check_manifest(root).values() for entry in group] + + +def test_release_evidence_is_canonical_and_contains_only_public_release_inputs(tmp_path): + root = _root(tmp_path) + evidence = build_evidence( + root, _dist(root), commit=COMMIT, tag=TAG, sbom=_sbom(root), verified_checks=_check_ids(root) + ) + + first = canonical_json_bytes(evidence) + second = canonical_json_bytes( + build_evidence( + root, root / "dist", commit=COMMIT, tag=TAG, + sbom=root / "release-evidence" / "engraphis-1.2.3.cdx.json", + verified_checks=_check_ids(root), + ) + ) + assert first == second + assert json.loads(first) == evidence + assert evidence["format"] == "engraphis-release-evidence/2" + assert evidence["package"] == {"name": "engraphis", "version": "1.2.3"} + assert evidence["commit"] == COMMIT + assert evidence["tag"] == TAG + assert [item["filename"] for item in evidence["artifacts"]] == [ + "engraphis-1.2.3-py3-none-any.whl", "engraphis-1.2.3.tar.gz" + ] + assert evidence["artifacts"][0]["sha256"] == hashlib.sha256(b"wheel").hexdigest() + assert [item["path"] for item in evidence["source_inputs"]] == [ + "pyproject.toml", "LICENSE", "NOTICE" + ] + assert evidence["checks"]["evaluations"][0]["inputs"][0]["path"] == ( + "eval/datasets/sample.jsonl" + ) + assert [ + item["path"] for item in evidence["checks"]["evaluations"][2]["inputs"] + ] == [ + "eval/datasets/sample.jsonl", + "eval/datasets/graph_multihop.jsonl", + ] + assert evidence["sbom"]["filename"] == "engraphis-1.2.3.cdx.json" + assert evidence["sbom"]["format"] == "CycloneDX" + assert evidence["provenance"]["builder"]["sbom_generator"]["version"] == "7.3.0" + assert evidence["provenance"]["builder"]["job"] == "release-evidence" + assert evidence["provenance"]["builder"]["completed_gate_jobs"] == [ + "build", "python-matrix", "browser-accessibility", "docker-smoke" + ] + assert len(evidence["limitations"]) == 3 + assert "exported_at" not in evidence + + +def test_release_evidence_fails_closed_when_checks_are_missing_or_unknown(tmp_path): + root = _root(tmp_path) + with pytest.raises(EvidenceError, match="verified checks"): + build_evidence(root, _dist(root), commit=COMMIT, tag=TAG, sbom=_sbom(root), verified_checks=["ruff"]) + with pytest.raises(EvidenceError, match="unexpected"): + build_evidence( + root, root / "dist", commit=COMMIT, tag=TAG, sbom=_sbom(root), + verified_checks=_check_ids(root) + ["made-up"], + ) + + +@pytest.mark.parametrize( + ("filename", "message"), + [ + ("engraphis-1.2.3-token.whl", "unsafe non-package file"), + ("engraphis-1.2.3.tar.gz", "unsafe non-package file"), + ], +) +def test_release_evidence_rejects_unsafe_distribution_inputs(tmp_path, filename, message): + root = _root(tmp_path) + dist = root / "dist" + dist.mkdir() + (dist / filename).write_bytes(b"candidate") + if filename.endswith(".tar.gz"): + (dist / "notes.txt").write_text("not a package") + with pytest.raises(EvidenceError, match=message): + build_evidence(root, dist, commit=COMMIT, tag=TAG, sbom=_sbom(root), verified_checks=_check_ids(root)) + + +def test_release_evidence_rejects_secret_like_values_even_in_package_filenames(tmp_path): + root = _root(tmp_path) + dist = _dist(root) + (dist / ("engraphis-1.2.3-sk_" + "a" * 16 + ".whl")).write_bytes(b"not safe") + with pytest.raises(EvidenceError, match="secret-like values"): + build_evidence(root, dist, commit=COMMIT, tag=TAG, sbom=_sbom(root), verified_checks=_check_ids(root)) + + +def test_release_evidence_fails_closed_for_unmatched_tags_and_invalid_sboms(tmp_path): + root = _root(tmp_path) + dist = _dist(root) + with pytest.raises(EvidenceError, match="tag"): + build_evidence(root, dist, commit=COMMIT, tag="v1.2.4", sbom=_sbom(root), verified_checks=_check_ids(root)) + + sbom = root / "release-evidence" / "engraphis-1.2.3.cdx.json" + sbom.write_text('{"bomFormat":"not-cyclonedx"}', encoding="utf-8") + with pytest.raises(EvidenceError, match="CycloneDX"): + build_evidence(root, dist, commit=COMMIT, tag=TAG, sbom=sbom, verified_checks=_check_ids(root)) + + +@pytest.mark.skipif(shutil.which("cyclonedx-py") is None, reason="release-only CycloneDX tool") +def test_release_environment_command_emits_a_cyclonedx_sbom(tmp_path): + output = tmp_path / "engraphis.cdx.json" + result = subprocess.run( + [ + "cyclonedx-py", "environment", "--output-reproducible", "--of", "JSON", + "--pyproject", "pyproject.toml", "-o", str(output), + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["bomFormat"] == "CycloneDX" + assert isinstance(payload["components"], list) + + +def test_release_workflow_publishes_evidence_separately_from_package_artifacts(): + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8") + build = workflow.split(" build:\n", 1)[1].split(" python-matrix:\n", 1)[0] + evidence_job = workflow.split(" release-evidence:\n", 1)[1].split(" publish:\n", 1)[0] + browser_job = workflow.split(" browser-accessibility:\n", 1)[1].split(" docker-smoke:\n", 1)[0] + github_release = workflow.split(" github-release:\n", 1)[1].split( + " github-release-repair:\n", 1 + )[0] + repair = workflow.split(" github-release-repair:\n", 1)[1] + + assert "cyclonedx-bom==7.3.0" in workflow + assert "cyclonedx-py environment --output-reproducible --of JSON" in workflow + assert "python scripts/release_evidence.py --dist dist --commit \"$GITHUB_SHA\"" in workflow + assert "--tag \"$GITHUB_REF_NAME\"" in workflow + assert "--sbom \"$sbom\"" in workflow + assert "--verified-check retrieval-ablation" in workflow + for check_id in ( + "privacy-boundary", "token-efficiency", "benchmark-schema-evidence", "browser-e2e", + "dependency-audit", "container-smoke", + ): + assert "--verified-check " + check_id in evidence_job + assert "needs: [build, python-matrix, browser-accessibility, docker-smoke]" in evidence_job + assert "name: Download distributions" in evidence_job + assert "npm run test:e2e" in browser_job + assert "Generate public release evidence" not in build + assert "needs: release-evidence" in workflow + assert "name: public-release-evidence" in workflow + assert "path: release-evidence/" in workflow + assert "Download public release evidence" in github_release + assert "dist/* release-evidence/release-evidence.json release-evidence/*.cdx.json" in github_release + assert "--name public-release-evidence" in repair + assert "dist/* release-evidence/release-evidence.json release-evidence/*.cdx.json" in repair + + +def test_receipt_export_has_a_stable_canonical_verification_view(): + service = MemoryService.create(":memory:") + service.remember("The production deploy window is Friday.", workspace="acme") + + first = service.export_receipts(workspace="acme") + second = service.export_receipts(workspace="acme") + + assert first["verification"]["valid"] is True + assert canonical_json_bytes(first) == canonical_json_bytes(second) + encoded = canonical_json_bytes(first).decode("utf-8") + assert "production deploy window" not in encoded + assert "acme" not in encoded From 33341d92e23a245dab4f0d477a5cf34a0d989a97 Mon Sep 17 00:00:00 2001 From: Jaixii Date: Thu, 30 Jul 2026 00:30:07 -0400 Subject: [PATCH 03/21] fix(hosted): harden Pro Team and Railway customer flows --- .github/workflows/ci.yml | 3 +- Dockerfile | 5 +- deploy/railway-template.json | 5 + docker-compose.yml | 6 +- docs/HOSTING_RAILWAY.md | 10 + docs/RAILWAY_TEMPLATE.md | 2 + docs/dashboard-button-qa.md | 55 +++ engraphis/classic_assets/dashboard.css | 1 + engraphis/classic_assets/dashboard.js | 50 +-- engraphis/cloud_features.py | 6 +- engraphis/cloud_session.py | 28 +- engraphis/dashboard_assets/index.html | 34 +- engraphis/dashboard_assets/ledger.css | 16 +- engraphis/dashboard_assets/ledger.js | 378 ++++++++++++++++++++- engraphis/inspector/app.py | 6 + engraphis/static/dashboard.css | 1 + engraphis/static/dashboard.js | 50 +-- tests/e2e/commercial.spec.js | 105 +++--- tests/e2e/ledger.spec.js | 129 ++++++- tests/test_cloud_features.py | 10 + tests/test_cloud_session.py | 55 +++ tests/test_customer_node_hardening.py | 83 +++++ tests/test_dashboard_auth_placement.py | 98 ++++-- tests/test_dashboard_button_regressions.py | 69 ++++ tests/test_dashboard_v2.py | 135 ++++++++ tests/test_graph_explorer_v2.py | 54 ++- tests/test_hosted_plan_resolution.py | 38 ++- tests/test_inspector_pro.py | 2 +- tests/test_llm_dashboard.py | 1 + tests/test_pro_cta.py | 45 +++ tests/test_release_infrastructure.py | 45 ++- 31 files changed, 1346 insertions(+), 179 deletions(-) create mode 100644 docs/dashboard-button-qa.md create mode 100644 tests/test_customer_node_hardening.py create mode 100644 tests/test_dashboard_button_regressions.py create mode 100644 tests/test_pro_cta.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f2aabf5..6698426a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,9 +161,10 @@ jobs: run: | python -m pip install --upgrade pip build pip-audit python -m build + python scripts/verify_distribution_contents.py dist/* python -m venv .audit-venv .audit-venv/bin/python -m pip install --upgrade "pip>=26.1.2" "setuptools>=83" .audit-venv/bin/python -m pip install dist/*.whl AUDIT_SITE=$(.audit-venv/bin/python -c "import site; print(site.getsitepackages()[0])") python -m pip_audit --path "$AUDIT_SITE" - .audit-venv/bin/python -c "import engraphis; print('wheel import OK')" + .audit-venv/bin/python -c "import engraphis, eval.harness; print('wheel imports OK')" diff --git a/Dockerfile b/Dockerfile index c6425a83..4c131703 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,9 @@ ENV PYTHONUNBUFFERED=1 \ # ONCE, not on every cold container. A fresh in-container download blocks startup and # can lose the healthcheck race; caching on the volume makes subsequent boots instant. HF_HOME=/data/.cache/huggingface \ - # Customer license / trial / machine-id / lease state. Kept on the /data volume - # (not the container's ephemeral home) so activation and device binding survive. + # Customer-side cloud session and entitlement display cache. Keep it on /data rather + # than the container's ephemeral home so reconnects do not lose rotated credentials. + # License issuance, trial state, leases, and revocations remain private services. ENGRAPHIS_STATE_DIR=/data/.engraphis WORKDIR /app diff --git a/deploy/railway-template.json b/deploy/railway-template.json index a66fa4bf..077721b7 100644 --- a/deploy/railway-template.json +++ b/deploy/railway-template.json @@ -41,6 +41,11 @@ "value": "*", "required": true }, + "ENGRAPHIS_DASHBOARD_URL": { + "value": "https://${{RAILWAY_PUBLIC_DOMAIN}}", + "prompt": "The Railway public domain is used for the dashboard and MCP HTTP origin allow-list. Override this with your HTTPS custom domain after it is active.", + "required": false + }, "ENGRAPHIS_CLOUD_CONTROL_URL": { "value": "https://api.engraphis.com", "required": false diff --git a/docker-compose.yml b/docker-compose.yml index 19674c71..a023d8eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,11 +32,9 @@ services: # safe only because the published host port above is loopback-only. ENGRAPHIS_LOCAL_TRUSTED_PEERS: 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fc00::/7 ENGRAPHIS_DB_PATH: /data/engraphis.db - # Persist license/trial/machine-id/lease + the revocation registry on the volume. + # Persist the customer-side cloud session and non-authoritative entitlement display + # cache on the volume. Issuance, trial state, leases, and revocations stay private. ENGRAPHIS_STATE_DIR: /data/.engraphis - # Relay/registry DB (issued keys + revocations) — on the persistent volume so a - # revoked key STAYS revoked across redeploys. - ENGRAPHIS_RELAY_DB: /data/.engraphis/relay.db volumes: - engraphis-data:/data restart: unless-stopped diff --git a/docs/HOSTING_RAILWAY.md b/docs/HOSTING_RAILWAY.md index 5cf510d5..1f119036 100644 --- a/docs/HOSTING_RAILWAY.md +++ b/docs/HOSTING_RAILWAY.md @@ -25,6 +25,16 @@ Set `ENGRAPHIS_FORWARDED_ALLOW_IPS=*` only when the container is reachable exclu Railway's trusted proxy. Set the dashboard's public URL where the runtime supports it, terminate TLS at the platform edge, and keep the volume private. +The published template derives `ENGRAPHIS_DASHBOARD_URL` from Railway's generated public domain. +That lets the dashboard's MCP-over-HTTP endpoint accept the public dashboard origin without +loosening its host/origin allow-list. If you attach a custom domain, override it with that domain's +canonical HTTPS URL after Railway has activated the domain; do not use an internal Railway domain +or a URL containing credentials. + +Do not add Resend (or any other email-provider) credentials to this customer node. The public +runtime has no transactional-email sender, verification, invitation, or billing-email service; +those systems remain in the official hosted control plane. + ## Connect to hosted Pro/Team services Complete onboarding through the official Engraphis Cloud dashboard, then configure only the diff --git a/docs/RAILWAY_TEMPLATE.md b/docs/RAILWAY_TEMPLATE.md index d9a7000f..f9c4ab5d 100644 --- a/docs/RAILWAY_TEMPLATE.md +++ b/docs/RAILWAY_TEMPLATE.md @@ -10,6 +10,8 @@ issuer, relay, managed compute, Auto Dreaming, Auto Consolidation, or Team ident - Service mode: `customer`. - Persistent volume: `/data`. - Health check: `/api/ready`. +- `ENGRAPHIS_DASHBOARD_URL` derived from Railway's generated public domain (override it with the + canonical HTTPS custom domain once one is active so public MCP origin checks remain strict). - Generated local API bearer supplied as `ENGRAPHIS_API_TOKEN`. - No vendor signer, billing, mail, Team-admin, relay-storage, or worker secrets. diff --git a/docs/dashboard-button-qa.md b/docs/dashboard-button-qa.md new file mode 100644 index 00000000..5a75c582 --- /dev/null +++ b/docs/dashboard-button-qa.md @@ -0,0 +1,55 @@ +# Dashboard button QA + +Date: 2026-07-29 +Scope: v2 Ledger (`/`) and legacy Classic (`/classic`) dashboards. + +## Test setup + +The manual pass used four parallel browser lanes and an isolated local v2 server on +`127.0.0.1:8701` with deterministic embeddings. The fixture contained the `demo` and +`beta` workspaces plus representative memories, graph data, provenance, timeline, and +consolidation state. The four lanes covered: + +- primary Ledger navigation, memory creation, grounded Ask, and theme controls; +- Library, import/editor actions, and empty-form behavior; +- Graph & Relations, Provenance, Manage, exports, saved views, and switches; +- broad regression including Classic and responsive/mobile keyboard behavior. + +## Button coverage + +The pass exercised the primary navigation, workspace selector, dashboard/theme switcher, +New memory, Save/Close, memory card actions, Import files, grounded answer, provenance +trace, timeline/history, supersessions, all Provenance tabs, all Graph tabs/styles/layouts/ +palettes/saved views/layers/toggles/actions/exports, all Manage tabs, workspace create and +workspace actions, consolidation preview and commit confirmation, plan comparison, and +Classic navigation/mobile-nav controls. + +## Failures found and fixed + +1. **Empty Save memory was silent.** Native form validation prevented the JavaScript + handler from running, leaving the editor open with no explanation. The editor now uses + explicit validation, an alert-region error, `aria-invalid`, focus on the content field, + and a status announcement. +2. **Closing the modern editor lost focus.** Close now returns focus to the button or card + that opened the editor, with a safe New memory fallback. +3. **Empty Ask, Provenance, Timeline/Supersessions, and workspace-create actions were + silent for the same native-validation reason.** These forms now use custom validation + messages and focus the relevant field. Successful submissions clear the prior status + message so an old validation error cannot remain beside a successful result. +4. **Classic mobile Escape closed the menu without reliably returning focus.** Escape now + closes the menu through the shared focus-restoring path. + +## Environment notes + +- One parallel lane could not start against the repository's default database because that + existing database is schema version 5 while this checkout supports schema version 4. + This is an environment/data compatibility issue, not a dashboard button failure. The + isolated schema-4 fixture started and exercised the UI successfully. +- The browser harness did not expose programmatic download events for the PNG/JSON export + anchors, but the dashboard status confirmed both exports completed. No application + console errors were observed during the manual pass. + +## Regression checks + +The focused static regression checks live in +`tests/test_dashboard_button_regressions.py` and cover each repaired failure mode. diff --git a/engraphis/classic_assets/dashboard.css b/engraphis/classic_assets/dashboard.css index 3d0d2c7d..9d9046db 100644 --- a/engraphis/classic_assets/dashboard.css +++ b/engraphis/classic_assets/dashboard.css @@ -271,6 +271,7 @@ body{ .empty{padding:var(--space-6) 0;color:var(--color-text-dim);line-height:1.5;text-align:left} .empty .btn{margin-top:var(--space-3)} .upgrade-panel{max-width:720px;padding:var(--space-6);border:var(--rule-strong) solid var(--color-accent);background:linear-gradient(135deg,var(--color-accent-bg),transparent 68%)}.upgrade-panel-kicker,.upgrade-panel-benefits-title{color:var(--color-accent);font-family:var(--mono);font-size:var(--text-micro);font-weight:600;letter-spacing:.08em}.upgrade-panel h2{margin:var(--space-2) 0;font-size:var(--text-xl);letter-spacing:-.02em;color:var(--color-text)}.upgrade-panel-lede{max-width:620px;margin:0;color:var(--color-text-dim)}.upgrade-panel-price{margin-top:var(--space-4);color:var(--color-text);font-size:var(--text-lg);font-weight:600}.upgrade-panel-benefits{margin-top:var(--space-4);padding-top:var(--space-4);border-top:var(--rule) solid var(--color-border)}.upgrade-panel-benefits ul{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--space-2) var(--space-5);margin:var(--space-3) 0 0;padding:0;list-style:none}.upgrade-panel-benefits li{display:flex;gap:var(--space-2);color:var(--color-text-muted);font-size:var(--text-sm);line-height:1.35}.upgrade-panel-benefits li::before{color:var(--color-accent);content:"✓"}.upgrade-panel-trial{margin:var(--space-4) 0 0;color:var(--color-text-dim);font-size:var(--text-sm)}.upgrade-panel-actions{display:flex;gap:var(--space-2);margin-top:var(--space-4);flex-wrap:wrap}.upgrade-panel-actions .btn{margin-top:0}@media(max-width:640px){.upgrade-panel{padding:var(--space-4)}.upgrade-panel-benefits ul{grid-template-columns:1fr}} +.pro-support-copy{margin-top:10px;padding:10px 12px;border-left:2px solid var(--color-accent);background:var(--color-accent-bg);color:var(--color-text-dim);font-size:var(--text-sm);line-height:1.45}.pro-support-copy strong{color:var(--color-text)} .hosted-opportunity{display:grid;max-width:900px;grid-template-columns:minmax(0,1.2fr) minmax(260px,.8fr);gap:var(--space-5);padding:var(--space-6);border:var(--rule-strong) solid var(--color-accent);background:linear-gradient(135deg,var(--color-accent-bg),transparent 65%)}.hosted-opportunity-kicker,.hosted-opportunity-preview-label,.hosted-opportunity-card span{color:var(--color-accent);font-family:var(--mono);font-size:var(--text-micro);font-weight:600;letter-spacing:.08em}.hosted-opportunity-kicker span{color:var(--color-text-dim)}.hosted-opportunity h2{max-width:14ch;margin:var(--space-2) 0 var(--space-3);color:var(--color-text);font-family:var(--font-display);font-size:var(--text-xl);letter-spacing:-.02em;line-height:1.05}.hosted-opportunity-lede{max-width:58ch;margin:0;color:var(--color-text-muted);font-size:var(--text-sm);line-height:1.55}.hosted-opportunity-next{margin:var(--space-4) 0 0;color:var(--color-text);font-size:var(--text-sm);font-weight:600;line-height:1.45}.hosted-opportunity-actions{display:flex;gap:var(--space-2);margin-top:var(--space-4);flex-wrap:wrap}.hosted-opportunity-preview{display:flex;min-width:0;flex-direction:column;gap:var(--space-2);padding:var(--space-4);border:var(--rule) solid var(--color-border);background:color-mix(in srgb,var(--color-raised) 72%,transparent)}.hosted-opportunity-preview-label{margin-bottom:var(--space-1);color:var(--color-text-dim)}.hosted-opportunity-card{padding:var(--space-3);border-left:var(--rule-strong) solid var(--color-accent);background:var(--color-panel)}.hosted-opportunity-card p{margin:var(--space-2) 0 0;color:var(--color-text-muted);font-size:var(--text-xs);line-height:1.45}.hosted-opportunity-privacy{grid-column:1/-1;padding-top:var(--space-3);border-top:var(--rule) solid var(--color-border);color:var(--color-text-dim);font-size:var(--text-xs);line-height:1.5}.hosted-opportunity-privacy strong{color:var(--color-text-muted)}@media(max-width:720px){.hosted-opportunity{grid-template-columns:1fr;padding:var(--space-4)}.hosted-opportunity h2{max-width:none}.hosted-opportunity-privacy{grid-column:auto}} /* Route compositions inherit the same ledger geometry. */ diff --git a/engraphis/classic_assets/dashboard.js b/engraphis/classic_assets/dashboard.js index 6f5f6932..0e99cdad 100644 --- a/engraphis/classic_assets/dashboard.js +++ b/engraphis/classic_assets/dashboard.js @@ -192,8 +192,9 @@ function fmtDay(epoch){const n=Number(epoch)||0;if(!(n>0))return '';try{const d= prefers ENGRAPHIS_PRO_UPGRADE_URL, so where the portal and the checkout are configured separately it is the Pro checkout under a neutral name. LIC.account_url resolves the generic value directly; the fallback only matters against a build that predates it. */ -function hostedAccountUrl(){return safeUrl((LIC&&LIC.account_url)||(LIC&&LIC.upgrade_url))} -function hostedPlanUrl(plan,trial,interval){const cadence=interval==='annual'?'annual':'monthly',key=plan+'_'+cadence+'_upgrade_url',raw=(LIC&&(LIC[key]||(plan==='team'?LIC.team_upgrade_url:LIC.pro_upgrade_url)))||(LIC&&LIC.upgrade_url);const safe=safeUrl(raw);if(!safe||safe==='#')return '#';try{const url=new URL(safe,location.href);if(plan==='pro'||plan==='team')url.searchParams.set('plan',plan);url.searchParams.set('interval',cadence);if(!url.hash)url.hash='billing';if(trial)url.searchParams.set('trial',plan);return url.href}catch(e){return safe}} +function withCtaAttribution(raw,content,medium){const safe=safeUrl(raw);if(!safe||safe==='#')return '#';try{const url=new URL(safe,location.href);url.searchParams.set('utm_source','engraphis');url.searchParams.set('utm_medium',medium||'product');url.searchParams.set('utm_campaign','pro_conversion');url.searchParams.set('utm_content',content||'plans');return url.href}catch(e){return safe}} +function hostedAccountUrl(content){return withCtaAttribution((LIC&&LIC.account_url)||(LIC&&LIC.upgrade_url),content||'account','product')} +function hostedPlanUrl(plan,trial,interval,content){const cadence=interval==='annual'?'annual':'monthly',key=plan+'_'+cadence+'_upgrade_url',raw=(LIC&&(LIC[key]||(plan==='team'?LIC.team_upgrade_url:LIC.pro_upgrade_url)))||(LIC&&LIC.upgrade_url);const safe=safeUrl(raw);if(!safe||safe==='#')return '#';try{const url=new URL(safe,location.href);if(plan==='pro'||plan==='team')url.searchParams.set('plan',plan);url.searchParams.set('interval',cadence);if(!url.hash)url.hash='billing';if(trial)url.searchParams.set('trial',plan);return withCtaAttribution(url.href,content||plan,'product')}catch(e){return safe}} /* Why is this feature locked? One sentence per access state, so the panel never claims a trial the customer cannot start nor blames billing for a trial that simply ran out. This is DENIAL copy: every caller reaches it because a hosted request was refused. The @@ -216,13 +217,15 @@ function teamTeaserNote(){const ends=licTrialEnds(); if(licPlanKey()!=='team'||!licAccessLive())return lockReason(true); if(licAccessState()==='trial')return `Your free trial includes Team${ends?` until ${esc(ends)}`:''}. Organizations, roles, and seats are managed in Engraphis Cloud.`; return 'Your TEAM subscription includes this. Organizations, roles, and seats are managed in Engraphis Cloud.'} -function unlockHtml(feature,plan){const url=hostedPlanUrl(plan,false,'monthly'),annualUrl=hostedPlanUrl(plan,false,'annual'),trialUrl=hostedPlanUrl(plan,true,'monthly'),team=plan==='team';const offerTrial=licTrialAvailable();const trial=team?'Start hosted Team trial':'Start hosted Pro trial';const purchase=team?'Purchase Team license':'Purchase Pro license';const price=team?'$20 per seat/month or $200 per seat/year':'$10/month or $100/year';const detail=lockReason(team);const benefits=team?['Everything in Pro','Hosted organizations, invitations, and named seats','Roles, scoped credentials, and Team audit history']:['Hosted Cloud Sync across your installations','Growth, retention, decay, and entity Analytics','Auto Consolidation with hosted retention policies','Auto Dreaming with reviewable managed proposals','Priority support'];return `
ENGRAPHIS ${team?'TEAM':'PRO'}

Unlock ${esc(feature)} and more

Make the local memory engine work across your installations—and keep improving without manual upkeep.

${price}
Your license unlocks
    ${benefits.map(item=>`
  • ${esc(item)}
    • `).join('')}

${detail}

${offerTrial?`${trial}`:''}${purchase}Annual option
`} +function hostedCta(plan,content,interval){const team=plan==='team',name=team?'Team':'Pro',state=licAccessState(),current=licPlanKey();if(state==='lapsed')return {label:'Update billing',href:hostedAccountUrl(content||'account'),kind:'account'};if(licAccessLive()&&(current===plan||(current==='team'&&plan==='pro')))return {label:current==='team'&&team?'Open Team Cloud':'Open Engraphis Cloud',href:hostedAccountUrl(content||'account'),kind:'account'};const trial=licTrialAvailable()&&state==='inactive';return {label:trial?`Start ${TRIAL_DAYS}-day ${name} trial`:`Subscribe to ${name}`,href:hostedPlanUrl(plan,trial,interval||'monthly',content||plan),kind:trial?'trial':'subscribe'} } +function ctaLinkHtml(cta,className,content){return `${esc(cta.label)}`} +function unlockHtml(feature,plan){const team=plan==='team',name=team?'Team':'Pro',featureKey=`feature_${String(feature).toLowerCase().replace(/[^a-z0-9]+/g,'_')}`,primary=hostedCta(plan,featureKey),annual=primary.kind==='account'?'':{label:`Annual ${name} option`,href:hostedPlanUrl(plan,false,'annual',`${featureKey}_annual`),kind:'subscribe'},price=team?'$20 per seat/month or $200 per seat/year':'$10/month or $100/year',detail=lockReason(team),benefits=team?['Everything in Pro','Hosted organizations, invitations, and named seats','Roles, scoped credentials, and Team audit history']:['Hosted Cloud Sync across your installations','Growth, retention, decay, and entity Analytics','Auto Consolidation with hosted retention policies','Auto Dreaming with reviewable managed proposals','Priority support'],lede=team?'Team adds shared workspaces, named seats, roles, and remote agent access.':'Support continued Engraphis development with Pro. Your subscription helps cover hosted infrastructure and ongoing development while unlocking Cloud Sync, Analytics, Auto Consolidation, and Auto Dreaming across your installations.';return `
ENGRAPHIS ${name.toUpperCase()}

Unlock ${esc(feature)} and more

${lede}

${price}
Your license unlocks
    ${benefits.map(item=>`
  • ${esc(item)}
    • `).join('')}

${detail}

${ctaLinkHtml(primary,'btn btn-primary',name.toLowerCase())}${annual.href&&annual.href!=='#'?ctaLinkHtml(annual,'btn btn-ghost',`${featureKey}_annual`):''}
`} function startTrialPlan(plan){const url=hostedPlanUrl(plan,true);if(url==='#'){toast('Hosted signup URL is not configured','err');return}const link=document.createElement('a');link.href=url;link.target='_blank';link.rel='noopener';link.click()} function startTrial(){return startTrialPlan('pro')} function startTeamTrial(){return startTrialPlan('team')} /* The badge follows the access state, not the plan name. A plan name alone told a trialist they were a subscriber, and told a lapsed or expired customer nothing was wrong. */ -function updateLicBadge(){const bd=document.getElementById('lic-badge');if(!bd||!LIC)return;const st=licAccessState(),plan=licPlanName();bd.textContent=st==='trial'?'TRIAL':st==='trial_expired'?'TRIAL ENDED':st==='lapsed'?plan+' INACTIVE':st==='active'?plan:'LOCAL';bd.className='pill '+(licAccessLive()?'pill-accent':'pill-muted')} +function updateLicBadge(){const bd=document.getElementById('lic-badge');if(!bd||!LIC)return;const st=licAccessState(),plan=licPlanName(),trial=licTrialAvailable(),label=st==='trial'?'TRIAL':st==='trial_expired'?'GET PRO':st==='lapsed'?'BILLING':st==='active'?plan:trial?'TRY PRO':'GET PRO',aria=st==='active'?'Open Engraphis Cloud account':st==='lapsed'?'Update billing in hosted plan settings':trial?'Start the 3-day Pro trial in hosted plan settings':'Subscribe to Pro in hosted plan settings';bd.textContent=label;bd.className='pill '+(licAccessLive()?'pill-accent':'pill-muted');bd.setAttribute('aria-label',aria);bd.title=aria} function updateFeatureLocks(){ const has=f=>LIC&&(LIC.features||[]).includes(f); const apply=(id,feature,label,plan)=>{ @@ -245,8 +248,13 @@ function renderAnalytics(a,isPortfolio){const t=a.totals||{},f=a.decay_forecast| /* A consent-required response is a valuable moment to show the job Pro can do, not a dead end about configuration. A customer with live access must never be offered their own plan again: hosted features are on by default once their account is available. */ -function managedConsentHtml(feature){const automation=/automation/i.test(feature),live=licAccessLive(),trial=licTrialAvailable(),copy=automation?{eyebrow:'MEMORY MAINTENANCE',title:'Let your memory improve after you log off.',lede:'Turn repetitive cleanup into a steady, reviewable habit. Pro watches the rhythm of your workspace and brings the useful changes back for approval.',cards:[['CONSOLIDATE','Distill recurring work into durable knowledge on a cadence you control.'],['DREAM','Surface useful links after accumulation and idle time, before fresh context gets buried.'],['REVIEW','Every managed result is a proposal. Nothing silently rewrites your local memory.']]}:{eyebrow:'MEMORY INTELLIGENCE',title:'See the memory your team is about to lose.',lede:'Pro turns your local memory into an operating signal—so you can see what is growing, what is fading, and what is quietly shaping recall.',cards:[['GROWTH','Separate knowledge that compounds from activity that only accumulates.'],['RETENTION','Catch fading context before an important answer disappears from reach.'],['ENTITY SIGNAL','See the people, projects, and ideas organizing your workspace.']]};const accountUrl=hostedAccountUrl(),trialUrl=hostedPlanUrl('pro',true),purchaseUrl=hostedPlanUrl('pro'),actions=live?`Open Engraphis Cloud`:`${trial?`Start ${TRIAL_DAYS}-day Pro trial`:''}Purchase Pro license`,next=live?'Included in your Pro plan. Hosted insights and maintenance are on by default—nothing else to configure.':trial?`Start with ${TRIAL_DAYS} days of Pro. Hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.`:'Purchase Pro and hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.';return `
ENGRAPHIS PRO /${copy.eyebrow}

${copy.title}

${copy.lede}

${next}

${actions}
WHAT PRO IS WATCHING
${copy.cards.map(card=>`
${card[0]}

${card[1]}

`).join('')}
Your memory stays yours. Hosted work is automatic with Pro. Secret and session-scoped memories stay local.
`} +function managedConsentHtml(feature){const automation=/automation/i.test(feature),featureKey=`managed_${String(feature).toLowerCase().replace(/[^a-z0-9]+/g,'_')}`,live=licAccessLive(),trial=licTrialAvailable(),copy=automation?{eyebrow:'MEMORY MAINTENANCE',title:'Let your memory improve after you log off.',lede:'Turn repetitive cleanup into a steady, reviewable habit. Pro watches the rhythm of your workspace and brings the useful changes back for approval.',cards:[['CONSOLIDATE','Distill recurring work into durable knowledge on a cadence you control.'],['DREAM','Surface useful links after accumulation and idle time, before fresh context gets buried.'],['REVIEW','Every managed result is a proposal. Nothing silently rewrites your local memory.']]}:{eyebrow:'MEMORY INTELLIGENCE',title:'See the memory your team is about to lose.',lede:'Pro turns your local memory into an operating signal—so you can see what is growing, what is fading, and what is quietly shaping recall.',cards:[['GROWTH','Separate knowledge that compounds from activity that only accumulates.'],['RETENTION','Catch fading context before an important answer disappears from reach.'],['ENTITY SIGNAL','See the people, projects, and ideas organizing your workspace.']]};const primary=hostedCta('pro',featureKey),annual=primary.kind==='account'?'':{label:'Annual Pro option',href:hostedPlanUrl('pro',false,'annual',`${featureKey}_annual`),kind:'subscribe'},actions=`${ctaLinkHtml(primary,'btn btn-primary',featureKey)}${annual.href?ctaLinkHtml(annual,'btn btn-ghost',`${featureKey}_annual`):''}`,next=live?'Included in your Pro plan. Hosted insights and maintenance are on by default—nothing else to configure.':licAccessState()==='lapsed'?'Your subscription needs billing attention. Update billing to restore hosted insights and maintenance.':trial?`Start with ${TRIAL_DAYS} days of Pro. Hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.`:'Subscribe to Pro and hosted insights and maintenance come on automatically—no settings, toggles, or worker setup.';return `
ENGRAPHIS PRO /${copy.eyebrow}

${copy.title}

${copy.lede}

${next}

${actions}
WHAT PRO IS WATCHING
${copy.cards.map(card=>`
${card[0]}

${card[1]}

`).join('')}
Your memory stays yours. Hosted work is automatic with Pro. Secret and session-scoped memories stay local.
`} function managedConsentRequired(error){return error&&error.status===409&&error.detail&&error.detail.code==='consent_required'} +const CLOUD_PRIVACY_COPY='Engraphis Cloud must read the bounded snapshot you submit to produce results. It travels over HTTPS but is not end-to-end encrypted; secret and session-scoped memories stay local.'; +const EXTERNAL_LLM_PRIVACY_COPY='Memory text is sent to your configured LLM provider for processing under that provider’s terms. The provider must read that text to return extracted facts.'; +async function confirmCloudTransfer(title,summary,submit){return confirmAction(title,summary+'\n\nPrivacy: '+CLOUD_PRIVACY_COPY,submit||'Continue')} +const managedConsentHtmlBase=managedConsentHtml; +managedConsentHtml=function(feature){return managedConsentHtmlBase(feature).replace('',`
Privacy, by design. ${esc(CLOUD_PRIVACY_COPY)}
`)}; /* Only an unconfigured local installation may turn a 401 into trial signup. A revoked or expired Cloud session is also a 401, but ``trial.available`` is false there and it must remain a reconnect error instead of offering a trial the control plane rejects. */ @@ -276,9 +284,12 @@ async function saveAutomation(){const body={enabled:document.getElementById('au- async function runMaintenance(){const el=document.getElementById('au-result');if(el)el.innerHTML='';try{const d=await api('/maintenance/run?workspace='+encodeURIComponent(WS||''),{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({dry_run:true})});if(el)el.innerHTML=`PROPOSAL Hosted work was submitted for review.
${esc(JSON.stringify(d,null,2))}
`;toast('Managed proposal requested','ok')}catch(e){if(el)el.innerHTML=managedConsentRequired(e)?managedConsentHtml('Hosted Automation'):'
'+esc(e.message)+'
';toast(managedConsentRequired(e)?'Hosted Automation starts automatically with Pro':((e.status===402||e.status===501)?'Hosted Automation requires Pro or Team':e.message),'err')}} const runMaintenanceBase=runMaintenance; +const saveAutomationBase=saveAutomation; +saveAutomation=async function(){const enabled=document.getElementById('au-enabled');if(enabled&&enabled.checked&&!await confirmCloudTransfer('Save hosted policy','Saving this enabled policy uploads this workspace’s normal and sensitive memory content to Engraphis Cloud; secret and session-scoped rows stay local.','Save policy'))return;return saveAutomationBase()} let MAINTENANCE_PENDING=false; runMaintenance=async function(dry){ if(MAINTENANCE_PENDING)return; + if(!await confirmCloudTransfer('Request hosted proposal','This sends this workspace’s normal and sensitive memory content to Engraphis Cloud for a reviewable proposal; secret and session-scoped rows stay local.','Request proposal'))return; const buttons=Array.from(document.querySelectorAll('#automation-body button[data-onclick="h90"]')),labels=buttons.map(button=>button.textContent); MAINTENANCE_PENDING=true; buttons.forEach(button=>{button.disabled=true}); @@ -505,17 +516,7 @@ function licStateBanner(state,plan,ends,status){ if(state==='lapsed'){const note=LIC_STATUS_NOTE[status];return `
Your ${esc(plan||'hosted')} subscription is no longer active${note?esc(note.charAt(0).toUpperCase()+note.slice(1))+', so hosted':'Hosted'} features are locked until billing is up to date. Your local memories are unaffected. Open the account portal to restore access.
`} if(state==='inactive')return `
No hosted plan on this installationThe local memory engine is free and complete on its own. Cloud Sync, Analytics, Automation, and Team administration run in Engraphis Cloud.
`; return ''} -function licActionsHtml(state){ - if(licTrialAvailable())return `
`; - /* A lapsed customer is fixing an existing subscription, not shopping. Both actions go - to the plan-neutral account portal so a payment-method problem is never reframed as a - new Pro or Team purchase. */ - if(state==='lapsed')return ``; - const buy=state==='trial_expired'; - if(state==='active'){const label=licPlanKey()==='team'?'Open Team Cloud':'Open Pro Cloud';return ``} - const primary=buy?'Subscribe to Pro':'Open Pro Cloud'; - const secondary=buy?'Subscribe to Team':'Open Team Cloud'; - return ``} +function licActionsHtml(state){const pro=hostedCta('pro','license');if(state==='active'||state==='lapsed')return `
${ctaLinkHtml(pro,'btn btn-primary btn-sm','license')}
`;const team=hostedCta('team','license_team');return `
${ctaLinkHtml(pro,'btn btn-primary btn-sm','license')}${ctaLinkHtml(team,'btn btn-ghost btn-sm','license_team')}
`} function renderLicense(d){ const el=document.getElementById('lic-body');if(!el)return; const state=licAccessState(),raw=String(d.plan||'local').toLowerCase(); @@ -535,6 +536,8 @@ function renderLicense(d){ it. Emitted by /api/license since the plan resolver landed, and never shown until now — so "the dashboard says PRO" and "the cloud says PRO" could not be told apart. */ if(d.plan_source)h+=`
Plan source${esc(LIC_SOURCE_LABEL[d.plan_source]||d.plan_source)}${d.plan_checked_at?' · confirmed '+esc(fmtRel(d.plan_checked_at)):''}
`; + if(state==='active')h+=`
Thank you for supporting Engraphis. Your subscription helps fund hosted infrastructure and ongoing development.
`; + else if(state!=='lapsed')h+=`
Support continued Engraphis development with Pro. Your subscription helps cover hosted infrastructure and ongoing development while unlocking Cloud Sync, Analytics, Auto Consolidation, and Auto Dreaming.
`; h+=`
The local core remains free. Pro and Team capabilities execute in Engraphis Cloud. The email-confirmed, no-card trial lasts exactly ${TRIAL_DAYS} active days; private-service account grace is separate, capped at 24 hours, and never extends cloud access or restricts local MCP and dashboard use.
`; h+=licActionsHtml(state); el.innerHTML=h; @@ -542,20 +545,22 @@ function renderLicense(d){ async function exportWorkspace(){try{const d=await api('/export?workspace='+encodeURIComponent(WS||''));const blob=new Blob([JSON.stringify(d,null,2)],{type:'application/json'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='engraphis-export-'+Date.now()+'.json';a.click();URL.revokeObjectURL(a.href);toast('Exported','ok')}catch(e){toast(e.message,'err')}} /* Hosted Team is a service CTA; local identity and seat administration are not shipped. */ -async function loadTeam(){const el=document.getElementById('team-body');let url=licPlanKey()==='team'&&licAccessLive()?hostedAccountUrl():hostedPlanUrl('team');try{const st=await api('/auth/state');if(url==='#'&&st&&st.cloud_url)url=safeUrl(st.cloud_url)}catch(e){}const trialUrl=hostedPlanUrl('team',true);el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
${esc(teamTeaserNote())} Private-service account grace is capped at 24 hours, never extends Team access, and never restricts the free local core.
${licTrialAvailable()?`Start hosted Team trial`:''}Open Team Cloud
`} +async function loadTeam(){const el=document.getElementById('team-body'),teamCta=hostedCta('team','team_tab');try{const st=await api('/auth/state');if(teamCta.href==='#'&&st&&st.cloud_url)teamCta.href=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
${esc(teamTeaserNote())} Private-service account grace is capped at 24 hours, never extends Team access, and never restricts the free local core.
${ctaLinkHtml(teamCta,'btn btn-primary btn-sm','team_tab')}
`} /* health + settings */ function connectionContext(){const host=(location.hostname||'').toLowerCase();return host==='localhost'||host==='127.0.0.1'||host==='::1'||host.endsWith('.localhost')?'Local engine':'Remote customer node'} async function checkHealth(){const label=connectionContext();try{await api('/health');const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-ok');d.classList.remove('health-error')}if(t)t.textContent=label+' connected'}catch(e){const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-error');d.classList.remove('health-ok')}if(t)t.textContent=label+' unavailable'}} function loadSettings(){loadLicense();loadSyncStatus();loadHostedAgentAccess();loadLlmStatus();const s=document.getElementById('cfg-store');if(s)s.textContent=location.host} -async function loadLlmStatus(){const el=document.getElementById('llm-body');if(!el)return;try{const st=await api('/llm/status');const ok=st.configured;const badge=ok?'configured':'not configured';const keyLine=st.key_set?'API key set ✓':'No API key set';let modelSel='';let provSel='';el.innerHTML=`
Provider · Model${badge}
${provSel}${modelSel}
${keyLine} · extractor: ${esc(st.extractor)}
Add this to your .env and restart Engraphis:
LLM extraction${st.extractor_enabled?'ON':'OFF'}
While ON, ingested memory content is sent to your LLM provider for schema-validated extraction. OFF keeps everything on this machine.
`}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} +async function loadLlmStatus(){const el=document.getElementById('llm-body');if(!el)return;try{const st=await api('/llm/status');const ok=st.configured;const badge=ok?'configured':'not configured';const keyLine=st.key_set?'API key set ✓':'No API key set';let modelSel='';let provSel='';el.innerHTML=`
Provider · Model${badge}
${provSel}${modelSel}
${keyLine} · extractor: ${esc(st.extractor)}
Add this to your .env and restart Engraphis:
LLM extraction${st.extractor_enabled?'ON':'OFF'}
While ON, ingested memory content is sent to your LLM provider for schema-validated extraction. OFF disables extraction transfers only; retention supervision is configured separately.
`}catch(e){el.innerHTML='
'+esc(e.message)+'
'}} function onLlmProvChange(){const p=document.getElementById('llm-prov').value;const sel=document.getElementById('llm-model');const defs={openai:'gpt-4o-mini',anthropic:'claude-3-5-sonnet-20241022',google:'gemini-1.5-flash',openrouter:'openai/gpt-4o-mini'};if(sel&&defs[p]){sel.value=defs[p]}updateLlmSnippet()} function updateLlmSnippet(){const p=(document.getElementById('llm-prov')||{}).value||'openai';const m=(document.getElementById('llm-model')||{}).value||'';const ta=document.getElementById('llm-snippet');if(!ta)return;ta.value='ENGRAPHIS_LLM_PROVIDER='+p+'\nENGRAPHIS_LLM_MODEL='+m+'\nENGRAPHIS_LLM_API_KEY=\nENGRAPHIS_EXTRACTOR=llm_structured\n'} function copyLlmSnippet(){const ta=document.getElementById('llm-snippet');if(!ta)return;ta.select();try{navigator.clipboard.writeText(ta.value);toast('Copied .env snippet','ok')}catch(e){toast('Copy failed — select and Ctrl+C','err')}} -async function setLlmExtractor(on){try{const d=await api('/llm/extractor',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({enabled:!!on})});const ok=!!d.extractor_enabled;toast(ok?'LLM extraction turned on — new memories will be sent to your provider':'LLM extraction turned off — memories stay on this machine'+(d.persisted===false?' (could not save for restart)':''),ok?'ok':'muted');loadLlmStatus()}catch(e){toast(e.message,'err')}} +async function setLlmExtractor(on){try{const d=await api('/llm/extractor',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({enabled:!!on})});const ok=!!d.extractor_enabled;toast(ok?'LLM extraction turned on — new memories will be sent to your provider':'LLM extraction turned off — extractor transfers are disabled'+(d.persisted===false?' (could not save for restart)':''),ok?'ok':'muted');loadLlmStatus()}catch(e){toast(e.message,'err')}} async function testLlm(){const r=document.getElementById('llm-test-result');if(r){r.textContent='Testing…';setTone(r,'muted')}try{const d=await api('/llm/test',{method:'POST'});if(r){if(d.ok){const transient=d.auto_enabled&&d.persisted===false;r.textContent=(transient?'⚠ ':'✓ ')+'Connected — '+esc(d.provider)+'/'+esc(d.model)+(transient?' Extraction is active for this process, but the setting could not be saved for restart. Set ENGRAPHIS_EXTRACTOR=llm_structured and ENGRAPHIS_LLM_AUTO_EXTRACT=1 in the deployment environment.':'');setTone(r,transient?'red':'green')}else{r.textContent='✗ '+(d.error||'failed');setTone(r,'red')}}}catch(e){if(r){r.textContent='✗ '+esc(e.message);setTone(r,'red')}}} -async function loadHostedAgentAccess(){const el=document.getElementById('tokens-body');if(!el)return;let url=licPlanKey()==='team'&&licAccessLive()?hostedAccountUrl():hostedPlanUrl('team');try{const st=await api('/auth/state');if(url==='#'&&st&&st.cloud_url)url=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Per-member agent accounts, roles, named seats, and rotating device credentials are managed in Team Cloud, not by this local dashboard.
`} +async function loadHostedAgentAccess(){const el=document.getElementById('tokens-body');if(!el)return;const teamCta=hostedCta('team','agent_access');try{const st=await api('/auth/state');if(teamCta.href==='#'&&st&&st.cloud_url)teamCta.href=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Per-member agent accounts, roles, named seats, and rotating device credentials are managed in Team Cloud, not by this local dashboard.
${ctaLinkHtml(teamCta,'btn btn-primary btn-sm','agent_access')}Agent Connect guide
`} +const setLlmExtractorBase=setLlmExtractor; +setLlmExtractor=async function(on){if(on&&!await confirmAction('Turn on LLM extraction',EXTERNAL_LLM_PRIVACY_COPY,'Turn on'))return;return setLlmExtractorBase(on)}; /* Route by cause, exactly like loadAnalytics/loadAutomation. Rendering the purchase panel for every failure told a paying customer to buy the plan they already own whenever the network blipped or the cloud answered 5xx. Only 401/402/501 are billing answers. */ @@ -565,6 +570,9 @@ function syncRecoveryHtml(){return unlockHtml('Cloud Sync','pro')+`
Hosted relayCONNECTED
Relay storage and authorization run in Engraphis Cloud. This package contains only the customer client; it does not run a local relay or background scheduler.
${esc(status)}
`} async function syncNow(){const b=document.getElementById('sync-btn')||document.getElementById('sync-retry-btn');const original=b&&b.textContent;const s=document.getElementById('sync-status');if(b){b.disabled=true;b.textContent='Syncing…'}if(s)s.textContent='Contacting the cloud…';try{const d=await api('/sync/run',{method:'POST',headers:{'Content-Type':'application/json'},body:'{}'});const su=d.summary||{};toast('Synced — pushed '+(su.exported||0)+', '+(su.added||0)+' new from other devices','ok');await loadSyncStatus()}catch(e){if(e.status===401||e.status===402||e.status===403){const el=document.getElementById('sync-body');if(el)el.innerHTML=syncRecoveryHtml();toast(e.status===402?'Cloud Sync requires an active Pro or Team entitlement — open Engraphis Cloud to upgrade or renew.':'Cloud Sync authorization is no longer active — reconnect in Engraphis Cloud.','err');return}toast('Sync failed: '+e.message,'err');if(b){b.disabled=false;b.textContent=original||'Sync now'}if(s)s.textContent='Sync failed — try again.'}} +const syncNowBase=syncNow; +syncNow=async function(){if(!await confirmCloudTransfer('Sync shared workspaces','Cloud Sync sends eligible changes from your shared workspaces to Engraphis Cloud and receives authorized changes from your other installations; secret and session-scoped rows stay local.','Sync now'))return;return syncNowBase()} + /* ─── knowledge graph (force-graph + d3-force: compact defaults and selectable layouts) ─── */ let GRAPH=null, FG=null, GRAPH_ENGINE=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}, GRAPH_FULL=false, GRAPH_SCOPE_BEFORE_FULL=null; const GRAPH_PRESETS={ @@ -1545,7 +1553,7 @@ document.addEventListener('keydown',event=>{ if(memories&&memories.classList.contains('show')){closeEntityMems();return} const theme=document.getElementById('theme-menu'); if(theme&&theme.classList.contains('is-open')){closeThemeMenu();document.getElementById('theme-btn').focus();return} - if(document.querySelector('.app').classList.contains('mobile-nav-open')){closeMobileNav();document.getElementById('mobile-nav-toggle').focus();return} + if(document.querySelector('.app').classList.contains('mobile-nav-open')){closeMobileNav(true);return} if(document.getElementById('view-mem-editor').classList.contains('active'))closeMem(); }); diff --git a/engraphis/cloud_features.py b/engraphis/cloud_features.py index c2239f30..7d90e06a 100644 --- a/engraphis/cloud_features.py +++ b/engraphis/cloud_features.py @@ -351,7 +351,8 @@ def _build_managed_snapshot_locked(service: Any, workspace: str, *, status=413) rows = service.store.conn.execute( "SELECT id, title, content, mtype, scope, ingested_at, last_access, valid_from, " - "valid_to, expired_at, stability, importance, pinned, sensitivity, metadata " + "valid_to, valid_to_recorded_at, expired_at, subject_key, claim_kind, " + "stability, importance, pinned, sensitivity, metadata " "FROM memories WHERE workspace_id=? AND COALESCE(scope, 'workspace')!='session' " "ORDER BY ingested_at, id", (workspace_id,), @@ -397,7 +398,10 @@ def _build_managed_snapshot_locked(service: Any, workspace: str, *, "last_access": float(item.get("last_access") or item.get("ingested_at") or 0), "valid_from": float(item.get("valid_from") or 0), "valid_to": item.get("valid_to"), + "valid_to_recorded_at": item.get("valid_to_recorded_at"), "expired_at": item.get("expired_at"), + "subject_key": str(item.get("subject_key") or ""), + "claim_kind": str(item.get("claim_kind") or ""), "stability": float(item.get("stability") or 1), "importance": float(item.get("importance") or 0.5), "pinned": bool(item.get("pinned")), diff --git a/engraphis/cloud_session.py b/engraphis/cloud_session.py index ca555011..62fec3e4 100644 --- a/engraphis/cloud_session.py +++ b/engraphis/cloud_session.py @@ -440,6 +440,12 @@ def _declared_entitlement(response: object) -> dict: return {} plan = plan.strip().lower()[:_MAX_PLAN_CHARS] active = response.get("cloud_access_active") + # Compatibility is for an *omitted* field from a control plane that predates this + # disclosure. An explicitly malformed field is not an older-server response: treating + # ``"false"`` (or ``0``) as absent would take the optimistic compatibility path and + # render a paid entitlement live. Keep the last good persisted answer instead. + if "cloud_access_active" in response and not isinstance(active, bool): + return {} named = response.get("status") named = named.strip().lower() if isinstance(named, str) else "" declared = { @@ -916,5 +922,25 @@ def access_for_workspace( if key not in declared: updated.pop(key, None) updated.update(declared) - _save(updated) + try: + _save(updated) + except (OSError, RuntimeError) as exc: + # The control plane has already consumed ``refresh``. Leaving that stale + # value usable after a local write fault makes the next request replay it, + # which can revoke the credential family. Retire it in memory first (so this + # process cannot replay it even when the state mount remains broken), then + # make a best-effort persisted retirement for a fault that was transient. + # The original write is deliberately never retried: it contains a replacement + # credential that may have reached disk only partially on an exotic mount. + _UNUSABLE_REFRESHES.add(_refresh_identity(refresh)) + try: + _mark_refresh_unusable(saved, refresh) + except Exception: # noqa: BLE001 - the state store is already failing + pass + raise CloudSessionError( + "Engraphis Cloud refreshed this session but the rotated credential " + "could not be saved. Connect this installation again.", + status=409, + refresh_unusable=True, + ) from exc return access, organization_id, compute diff --git a/engraphis/dashboard_assets/index.html b/engraphis/dashboard_assets/index.html index dd6270fb..0d19f388 100644 --- a/engraphis/dashboard_assets/index.html +++ b/engraphis/dashboard_assets/index.html @@ -14,7 +14,7 @@