From eaa802b41b3d79bff88fb157e5419e23f5e4d219 Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 11:41:53 -0700 Subject: [PATCH 1/8] =?UTF-8?q?docs(research):=20Phase=201=20=E2=80=94=20m?= =?UTF-8?q?emory=20formation=20&=20query=20transformation=20R&D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read-only Phase 1 deliverables for the Engram memory-formation + query-transformation initiative: - 01 current-state map: ingestion→recall data flow, score semantics, data model, intervention points, existing LLM usage - 02 research memo: literature review + blind-spot pass with cited primary sources; separates published evidence from inference - 03 experiment spec: A–E ablation matrix, graduated corpora, retrieval + downstream metrics, saturation-aware reproducibility controls - README: exec summary, sequencing, open questions No code, data, or migrations touched. Prototypes remain unbuilt pending review. Co-Authored-By: Claude Opus 4 --- .../01-current-state-map.md | 212 ++++++++++++++++++ .../02-research-memo.md | 152 +++++++++++++ .../03-experiment-spec.md | 207 +++++++++++++++++ .../README.md | 82 +++++++ 4 files changed, 653 insertions(+) create mode 100644 docs/research/memory-formation-query-transform/01-current-state-map.md create mode 100644 docs/research/memory-formation-query-transform/02-research-memo.md create mode 100644 docs/research/memory-formation-query-transform/03-experiment-spec.md create mode 100644 docs/research/memory-formation-query-transform/README.md diff --git a/docs/research/memory-formation-query-transform/01-current-state-map.md b/docs/research/memory-formation-query-transform/01-current-state-map.md new file mode 100644 index 00000000..b2e4d667 --- /dev/null +++ b/docs/research/memory-formation-query-transform/01-current-state-map.md @@ -0,0 +1,212 @@ +# Current-State Map: Engram Memory Lifecycle (Ingestion → Recall) + +**Scope:** Read-only audit of Engram's `staging` branch. No code or data modified. +**Date:** 2026-08-23 +**Purpose:** Map the ingestion→recall data flow and identify intervention points for +(A) LLM-assisted memory formation at store time and (B) LLM-assisted query +transformation at recall time. + +> File/line references are as observed during the audit against `staging` +> (base commit `7af144b3`). Line numbers drift; treat them as anchors, re-grep +> before editing. + +--- + +## 1. Ingestion → Recall Data Flow + +### Ingestion entry points (memory create) +- `POST /v1/memories` — single create (`memory-core.controller.ts` ~L74) +- `POST /v1/memories/batch` — synchronous batch (~L95) +- `POST /v1/memories/batch/async` — async batch via BullMQ, returns `jobId` (~L118) +- `POST /v1/memories/bulk` — high-volume insert (via `bulk.dto`) + +### Write path (`memory-write.service.ts` `remember()` ~L66–330) +1. Fetch user context (accountId, display name) (~L78) +2. `ImportanceService.calculate()` from layer hint (~L95) +3. Confidence from `SOURCE_CONFIDENCE` map — `EXPLICIT_STATEMENT=1.0`, + `AGENT_OBSERVATION=0.7`, etc. (~L100) +4. Resolve temporal anchor — `observedAt` (speaker "now") vs `createdAt` (server), + `temporalAnchorSource` enum (~L157) +5. Insert `Memory` record (raw, layer, source, importance, confidence, tags, + metadata) (~L163) +6. Emit temporal warnings (HISTORICAL without anchor) (~L186) +7. **HyPE generation** — fire-and-forget hypothetical-prompt-embedding (~L198) +8. **Async extraction + embedding** via `EmbeddingQueueProducer`, or sync via + `MemoryPipelineService.extractAndEmbed()` (~L244) +9. Elasticsearch/BM25 index — fire-and-forget (~L308) +10. Contradiction detection via `CorrectionService` — fire-and-forget (~L297) + +### Extraction (`extraction.service.ts`) +- Calls **`LLMService.json()`** with `EXTRACTION_PROMPT_TEMPLATE` (temp 0.2) +- Produces 5W1H: who/what/when/where/why/how + topics[], entities[], memoryType, + per-field confidence +- Invoked inside `MemoryPipelineService.extractAndEmbed()` in parallel with + embedding generation (~L67) + +### Embedding generation +- `EmbedFacade.embedOne()` preferred (`memory/embedding.service.ts` ~L73); + `LLMService.embed()` is the legacy fallback (can diverge if + `EMBEDDING_PROVIDER` differs) +- Model ID via `resolveEmbeddingModelId()` (`embedding-model.util.ts` ~L14): + `EMBEDDING_MODEL ?? VECTOR_SEARCH_MODEL ?? 'bge-base'` +- Dimension validation: 1536 (openai-small), 3072 (openai-large), 768 (bge-base), + 384 (minilm), 768 (nomic) +- **Circuit breaker** (~L26–106): 5 consecutive failures → 1-minute block; + transient 503s (engram-embed backlog) don't count +- **Priority recall path** `generateForRecall()` (~L113): adds `X-Priority: recall` + header to skip the batch queue, 5s timeout + +### Vector storage +- Default **pgvector**: `memory_embeddings(memory_id, model_id, dimensions, + embedding)`, unique `(memory_id, model_id)` +- Optional **Pinecone** +- Per-model tables (`feat/embed-per-model-tables`): `EmbeddingOpenaiSmall`, + `EmbeddingBgeBase`, `EmbeddingMinilm`, `EmbeddingNomic` + (`schema.prisma` ~L254) + +--- + +## 2. Recall Path + +### Recall endpoints (`memory-query.controller.ts`) +- `POST /v1/memories/query` — **canonical** (~L102); `response_format=structured` + yields v2 shape +- `POST /v1/recall` — **deprecated** alias (~L264) +- `POST /v1/context` — session bootstrap (~L329) +- `POST /v1/recall/contextual` — mid-conversation w/ topic-shift detection (~L314) +- `POST /v1/memories/timeline` — chronological + temporal gap detection (~L417) + +### Core algorithm (`memory-query.service.ts` `recall()` ~L67–430) +1. `TemporalParserService` parses temporal intent ("last week", "years ago") (~L115) +2. `EmbeddingService.generateForRecall()` — priority flag, 5s timeout (~L131) +3. **Candidate retrieval:** + - **Temporal path** (~L155): adaptive window expansion (doubling span, 200ms + timeout, min 5 results); blends semantic + temporal + importance; rerank + pool of 120 + - **Standard path** (~L358): hybrid semantic + keyword. + Vector via pgvector `embedding <=> query`, `score = 1 - distance`. + BM25/tsvector rescue via `websearch_to_tsquery` (RRF k=60) + ILIKE fallback. +4. **Scoring composition** (~L350): + `blendedScore = semanticScore × importanceScore` (× temporal factor on temporal + path); `adjustedScore = blendedScore × recallWeight × importanceMultiplier` + (`importanceMultiplier = 0.4` if `importanceScore < 0.35` else 1.0) +5. **Reranking** (`memory-query-ranking.service.ts` ~L224): cross-encoder (TEI + reranker) normalizes to [0,1], then + `finalScore = (rerankerScore × 0.85 + importanceScore × 0.15) × sentimentPenalty`; + fallback blend uses cosine in place of reranker +6. **Graph recall merge** (~L71): `GraphRecallService` injects graph-traversed + memories with **×1.2 boost** on dedup hits +7. **Insight surfacing** (~L112): INSIGHT-layer memories with sim > 0.3, boost + `= semanticScore + importanceScore × 0.3` +8. **Deduplication** (SQL-level): `superseded_by_id IS NULL`, + `searchable IS NOT FALSE`, `embedding_status != 'DUPLICATE'`, + `is_duplicate_of IS NULL` + +### Response shapes (`memory.types.ts`) +- Legacy `QueryResult`: `{recallId, memories[], queryTokens, latencyMs, + multiQuery?, explanations?}` (~L78) +- Structured v2 `StructuredQueryResult`: typed `fact`, `source_session`, + `confidence`, `timestamp`, `memory_type`, optional `ChainOfNotePrompt` + +--- + +## 3. Score Semantics (⚠️ correctness smell) + +- **Cosine** (`pgvector.provider.ts` ~L234): `score = 1 - (embedding <=> query)` → + **[0,1]**, 1 = perfect match. +- **Keyword rescue** (`memory-query.service.ts` ~L446, ~L524): FTS-only hits are + **hard-coded to 1.25**; ILIKE-only hits to **1.1** — deliberately above the + [0,1] ceiling as penalty-free "exact-match agrees" overrides. +- **Graph recall** applies **×1.2**. +- **Net effect:** the 1.10/1.25 out-of-range scores the mnemon brief flagged, and + wrong distractors tying correct memories at 1.25, come from these overrides. + Scores are **relative sort ranks, not probabilities/thresholds.** +- **Client risk:** any consumer assuming a `[0,1]` threshold (e.g. a 0.5 cutoff) + breaks on rescue hits. **Recommendation:** document scores as relative ranks; + consider a separate calibrated confidence field. Worth a ticket regardless of + the R&D outcome. + +--- + +## 4. Data Model (what exists vs what formation needs) + +### `Memory` (`schema.prisma` ~L134–287) — selected fields +- Identity: `id`, `userId`, `projectId?`, `sessionId?` +- Content: `raw`, `layer` (IDENTITY/SESSION/FACT/EVENT/INSIGHT/CONSTRAINT/LESSON) +- Extraction: `memoryType`, `typeConfidence`, `factKeys[]` +- Importance: `importanceScore`, `effectiveScore`, `importanceHint` +- Confidence: `confidence`, `source` +- Temporal: `observedAt`, `temporalAnchorSource`, `eventTimes[]` +- Embedding: `embeddingId`, `embeddingModel`, `embeddingStatus` + (PENDING/EMBEDDED/DUPLICATE/FAILED), `isDuplicateOf?`, `parentMemoryId?` +- Lifecycle: `createdAt`, `updatedAt`, `deletedAt`, `ingestedAt`, + `consolidatedAt`, `lastRetrievedAt`, `usedCount`, `retrievalCount` +- Consolidation: `supersededById`, `consolidatedInto` +- Classification: `durability`, `userPinned`, `userHidden`, `safetyCritical` +- Graph: `outgoingEdges[]`, `incomingEdges[]` (`MemoryEdge`) +- Visibility: `visibility` (PRIVATE/SHARED/ACCOUNT), `tags[]`, `metadata` (JSON) +- Dedup: `contentHash` + +### Related +- `MemoryExtraction` — 5W1H + topics/factKeys/rawJson/memoryType/confidence +- `MemoryEmbedding` — `(memory_id, model_id, dimensions, embedding)` +- `MemoryEdge` (ENG-120) — `sourceMemoryId`, `targetMemoryId`, `edgeType` + (RELATED_TO / CONTRADICTS / REINFORCES / SUPERSEDES …), `confidence` +- `MemoryEventTime` — resolved temporal references + +### What formation would add +- Store formed/canonical text in `content` (DTO already accepts `content` + + `raw`), retaining `raw` for provenance — **no schema change required for a + union-storage MVP.** +- Optional: `metadata.formation = {formationScore, promptVersion, model, + suggestedRaw}` or a dedicated `MemoryFormation` table if we want it queryable. + +--- + +## 5. Likely Intervention Points + +| Intervention | File / method | Anchor | Trigger | +|---|---|---|---| +| Formation (store) | `src/memory/formation.service.ts` (**new**) `formMemory()` | — | after `Memory.create`, before embedding enqueue | +| Formation (extract) | `MemoryPipelineService.extractAndEmbed()` | insert after ~L72 | post-extraction, pre-embedding (**preferred** — has extraction context) | +| Query transform (recall) | `src/memory/query-transform.service.ts` (**new**) `transformQuery()` | — | in `recall()` after temporal parse (~L115), before `generateForRecall()` | +| Query transform (expand) | `MemoryQueryService.recall()` | insert after ~L328 | post-vector-search, pre-rerank | + +Both gate behind env flags (`ENABLE_MEMORY_FORMATION`, `ENABLE_QUERY_TRANSFORM`) +and follow the optional-`LLMService` injection pattern already used by +`InsightGeneratorService` / `ConsolidationService`. On flag=off or error, skip +the step and continue — no behavior change to the default path. + +--- + +## 6. Existing LLM Usage (formation precedents) + +1. **`ExtractionService.extract()`** — LLM 5W1H extraction (temp 0.2). This is + the natural seam for a formation refinement pass. +2. **`InsightGeneratorService.synthesizeWithLlm()`** + (`awareness/analysis/insight-generator.service.ts` ~L135) — pattern→INSIGHT + synthesis, **budget-aware / maxLlmCalls-gated**. Reuse this gating pattern for + formation cost control. +3. **`ConsolidationService.promoteRecurringPatterns()`** (~L68) — clustering + + optional canonicalization; promotion chains via `supersededById`. +4. **`MemoryDedupService.findDuplicateV2()`** (~L61) — deterministic vector + thresholds (0.93 auto-merge / 0.85 reinforce / 0.78 review); **no LLM**. + +**Takeaway:** Engram already spends LLM calls at store time (extraction) and in +consolidation. Formation is a *second refinement pass*, not net-new plumbing, and +a clean feature-flag precedent already exists. + +--- + +## Key implications for the R&D + +- The binding constraint the mnemon data exposes is **ranking under noise** and + **corpus hygiene** (duplicates/incompleteness), not representation poverty. The + existing hybrid+rerank+RRF stack is where our actual bottleneck lives. +- **Query transformation (C)** has a ready seam: `StructuredQueryResult.multiQuery` + + `explanations` + `X-Query-Id` already exist — instrument, don't rebuild. +- **Formation (B)** should follow **union storage** (raw ∪ derived), never replace + `raw` — consistent with both the schema (retain `raw`, fill `content`) and the + external negative result on artifact-replacement (see research memo §1, §3). +- The **1.25/1.10 score overrides** are a real correctness smell; file a ticket + independent of this initiative. diff --git a/docs/research/memory-formation-query-transform/02-research-memo.md b/docs/research/memory-formation-query-transform/02-research-memo.md new file mode 100644 index 00000000..0073d94d --- /dev/null +++ b/docs/research/memory-formation-query-transform/02-research-memo.md @@ -0,0 +1,152 @@ +# Research Memo: LLM-Assisted Memory Formation and Query Transformation in Engram + +**Audience:** Engram R&D / architecture decision +**Scope:** Research and analysis only. No code or data was modified. +**Date:** 2026-08-23 + +--- + +## 0. Framing and the evidence we already own + +We are evaluating two interventions in Engram's memory lifecycle: + +- **(A) LLM-assisted memory formation** at store time — spend an LLM call when a memory is written to produce a richer representation (canonical fact, retrieval synopsis, structured entities/relations, temporal scope, provenance, confidence, predicted queries). +- **(B) LLM-assisted query transformation** at recall time — rewrite/expand/decompose the user's query before retrieval. + +Our own prior evidence (mnemon benchmark, `/Users/beauxwalton/projects/mnemon`, `RESULTS.md` + `reports/*.json`) is the anchor and, notably, it is *not* about either intervention — it measures the value of *relevant* retrieved memory on executable coding output: + +- Qwen 3.8 27B, clean corpus: **0.820 vs 0.601, +0.219 lift**, 95% CI [0.117, 0.317], target memory rank-1 on 20/20 tasks. +- Qwen, 200 distractors: **+0.101 lift**, CI [0.014, 0.195], rank-1 on only 13/20. +- Gemma 4 31B, 200 distractors: **+0.117 lift**, CI [0.041, 0.203], rank-1 on 12/20. + +The brief also records four uncomfortable findings from the seeded/ablation runs: near-duplicate memories roughly **halved** the benefit; incomplete memories could **harm** output; the grader produced **out-of-range scores (1.10, 1.25)**; and wrong distractors sometimes **tied** correct memories at rank 1. I could confirm the headline lifts and the noisy-corpus rank degradation directly in `RESULTS.md`; the duplicate-halving, incomplete-harm, and >1.0 score figures are stated in the brief's prior-evidence summary and I did not independently re-derive them from the raw `reports/*.json` in the time available. They should be treated as reported-by-us, not yet re-verified here. + +**The single most important thing this data already tells us:** the binding constraint in our own system is *retrieval ranking under noise* (rank-1 fell from 20/20 to 12–13/20), and the biggest observed harms come from *corpus quality* (duplicates, incompleteness), not from a shortage of representation cleverness. Keep this in mind through the whole memo — it directly predicts which of the two interventions will pay off. + +--- + +## 1. Approaches reviewed — Memory Formation + +For each: mechanism, evidence/claims, limitations, source. **Published evidence** and **my inference** are separated explicitly. + +**Raw + derived multi-representation storage.** *Mechanism:* keep the verbatim text AND store derived artifacts (summary, entities, synthetic queries) alongside, retrieving over either. *Evidence:* the strongest recent controlled result argues *for keeping raw and against replacing it* — see "Verbatim Chunks" below. *Limitation:* storage and index bloat, and you must decide what to embed. *My inference:* this is the safe default; it dominates "replace raw with derived." + +**Proposition / atomic-fact decomposition.** *Mechanism:* an LLM rewrites a passage into self-contained atomic propositions, each embedded and retrieved as its own unit. *Evidence:* "Dense X Retrieval: What Retrieval Granularity Should We Use?" (Chen et al., arXiv:2312.06648, Dec 2023) reports proposition-level units outperform sentence/passage units for dense retrieval and downstream QA, because each unit is denser in query-relevant information. *Limitation:* decomposition is itself an LLM step that can drop or distort facts; propositions lose surrounding context; more units = more index. *Source:* Chen et al., 2023. + +**Entity-relation extraction (triples / knowledge graph).** *Mechanism:* extract (subject, relation, object) triples and store a graph, retrieving by graph traversal + vector. *Evidence:* used in Mem0g and Zep/Graphiti (below); helps multi-hop and temporal reasoning. *Limitation:* extraction error compounds; graph construction is expensive and brittle; a 1-hop graph did *not* recover the loss from discarding verbatim text in the CogCanvas ablation. *Source:* see Mem0/Zep. + +**Summarization-for-retrieval.** *Mechanism:* store an LLM summary as the retrievable representation. *Evidence:* helps when raw is long/noisy. *Limitation:* lossy — summaries discard the exact tokens a future query may need. *My inference:* summaries are a good *auxiliary* field, a poor *replacement*. + +**HyDE-style synthetic queries stored at index time ("predicted queries").** *Mechanism:* at store time, generate the questions this memory would answer, embed those, and index them so a real query matches a synthetic query. *Evidence:* no clean published isolation of *store-time* synthetic-query indexing that I could verify; it is the inverse of HyDE (Sec 2), which generates a hypothetical *document* at query time. *Limitation:* predicted queries are guesses; they can bias the memory toward anticipated (and away from novel) uses. *My inference:* plausibly useful for FAQ-shaped memory, unproven for open-ended agent memory. **Flag as unverified.** + +**Contextual Retrieval (Anthropic).** *Mechanism:* before embedding a chunk, prepend a 50–100 token LLM-generated context sentence situating the chunk in its parent document; index both contextual embeddings and contextual BM25. *Evidence (published):* Published **Sep 19, 2024**. Contextual embeddings alone cut top-20 retrieval failure **35%** (5.7%→3.7%); + contextual BM25 **49%** (→2.9%); + reranking **67%** (→1.9%). *Limitations (their own):* pointless under ~200k tokens (just stuff full content in context); results vary by embedding model, chunk size/overlap; reranking adds latency; "evaluate within your use case." *Source:* anthropic.com/engineering/contextual-retrieval, 2024-09-19. **This is the best-evidenced formation technique in the set** and is a formation-side (index-time) win, not a query-side one. + +**RAPTOR (hierarchical summaries).** *Mechanism:* recursively cluster + summarize chunks bottom-up into a tree; retrieve from multiple abstraction levels. *Evidence (published):* Sarthi et al., ICLR 2024 — SOTA on multi-step QA (e.g., QuALITY, NarrativeQA) vs standard chunk RAG. *Limitation:* tree build is expensive and static; summary nodes are lossy; benefit concentrates on *holistic/multi-hop* questions, less on point-lookup. *Source:* Sarthi et al., ICLR 2024. + +**MemGPT / Letta.** *Mechanism:* OS-style paging between in-context "main memory" and out-of-context external store; the LLM manages its own memory via tool calls. *Evidence (published):* Packer et al., arXiv:2310.08560 (Oct 2023) — extends effective context for long docs and multi-session chat. *Limitation:* it is a *control loop / context-management* pattern, not a representation recipe; adds many extra LLM calls and non-determinism. *Source:* Packer et al., 2023 (now the Letta framework). + +**Mem0.** *Mechanism:* at store time an LLM extracts salient facts and issues ADD/UPDATE/DELETE/NOOP against existing records; Mem0g adds an entity/triple graph. *Evidence (published):* arXiv:2504.19413 (ECAI 2025). Vendor-reported LOCOMO ~67% LLM-judge in the paper (later vendor blog figures ~92.5 LOCOMO / ~94.4 LongMemEval), ~1,764 tokens/conversation vs ~26k full-context, p95 search ~0.2s. *Limitation:* benchmarks are largely vendor-run; the LLM UPDATE/DELETE step is exactly where silent corruption/over-deletion can enter. *Source:* Chhikara et al., 2025. **Treat vendor numbers as claims, not independent evidence.** + +**Zep / Graphiti (temporal knowledge graph).** *Mechanism:* bi-temporal knowledge graph (tracks valid-time and system/ingestion-time; invalidates edges when facts change) synthesizing conversational + structured data. *Evidence (published):* Rasmussen et al., arXiv:2501.13956 (Jan 2025) — DMR 94.8% vs MemGPT 93.4%; LongMemEval up to +18.5% accuracy and ~90% lower latency vs baseline. *Limitation:* vendor-authored; graph construction cost/complexity; extraction errors propagate; DMR is near-saturated (94 vs 93 is a thin margin). *Source:* Rasmussen et al., 2025. + +**Cross-cutting caution — "Verbatim Chunks Beat Extracted Artifacts" (CogCanvas).** *Mechanism:* a *controlled ablation* — same retriever/reranker/reasoner/judge, swap only the stored representation: LLM-extracted typed artifacts vs verbatim chunks. *Evidence (published):* An, arXiv:2601.00821 (2026). Verbatim beat extracted artifacts by **+15.9 pts LoCoMo (43.9 vs 28.0)** and **+22.0 pts LongMemEval-S (67.4 vs 45.4)**; a 1-hop semantic graph did not recover the gap; five confound controls reproduced it; extracted-artifacts alone *never beat naive RAG*. The one design that matched verbatim was **chunks ∪ artifacts (union)**. *Why load-bearing:* this is the closest published analogue to Intervention A, and it is a *negative* result for "replace raw with LLM-derived structure." *Source:* An, 2026. + +--- + +## 2. Approaches reviewed — Query Transformation + +**Single query rewrite.** *Mechanism:* LLM rewrites one query into one cleaner query (spelling, context resolution, term normalization). *Evidence:* "Not All Queries Need Rewriting" (arXiv:2603.13301, 2026) — rewriting helps some queries and *hurts others*; harm co-occurs with reduced lexical alignment to the gold doc (rewriting swaps domain terms on already-good queries). Controlled conversational rewrite + last-turn concatenation gave consistent gains; retrieval-oriented keyword rewrites consistently hurt. *Limitation:* on already-well-matched queries it is net-negative. *Source:* 2026. + +**Multi-query expansion / RAG-Fusion.** *Mechanism:* generate N query variants, retrieve for each, fuse. *Evidence:* RAG-Fusion (Raudaschl, 2023/2024; GitHub Raudaschl/rag-fusion) reports robustness gains from diverse angles; the fusion step rests on RRF (below). *Limitation:* N× retrieval cost and latency; broadening the pool in the wrong direction just gives the reranker more wrong things to sort. *Source:* Raudaschl, 2023–24. + +**Query decomposition (multi-hop / least-to-most).** *Mechanism:* split a complex query into sub-queries, retrieve per sub-query, compose. *Evidence:* helps multi-hop QA (MultiHop-RAG, arXiv:2401.15391; "Question Decomposition for RAG," arXiv:2507.00355). "When Should Queries Be Decomposed?" (arXiv:2606.08577, 2026) finds it is *stage-* and *condition-dependent* — decomposition helps multi-condition retrieval but is not universally good. *Limitation:* over-decomposition fragments intent; more calls, more latency; wrong on single-hop queries. *Source:* 2024–2026. + +**Step-back prompting.** *Mechanism:* abstract the specific question to its general principle, retrieve/reason at that level, then descend. *Evidence:* Zheng et al. (Google DeepMind), arXiv:2310.06117 (Oct 9 2023) — e.g., +7%/+11% MMLU physics/chem, +27% TimeQA, +7% MuSiQue on the *reasoning* tasks tested. *Limitation:* the paper is about reasoning, not retrieval ranking per se; abstraction can retrieve too-general context for point lookups. *Source:* Zheng et al., 2023. + +**HyDE (hypothetical document embeddings).** *Mechanism:* LLM writes a hypothetical answer/document; embed *that* to find real neighbors; the encoder's bottleneck filters hallucinated specifics. *Evidence:* Gao et al., "Precise Zero-Shot Dense Retrieval without Relevance Labels," arXiv:2212.10496 (Dec 2022, ACL 2023) — beats unsupervised Contriever, near fine-tuned retrievers, zero-shot, cross-lingual. *Limitation:* strongest exactly where you *lack* a tuned retriever/labels; on a well-tuned domain retriever the gain shrinks, and it can inject wrong terms. *Source:* Gao et al., 2022/23. + +**Entity/keyword extraction for metadata filtering & self-query.** *Mechanism:* LLM extracts filters (source, date, type, entity) from the query and applies them as structured pre-filters. *Evidence:* Multi-Meta-RAG (Springer, 2024) — small-LLM metadata extraction restricting candidate sets improves multi-hop RAG; LangChain "self-query" is the widely-used reference implementation. *Limitation:* an over-tight filter can *exclude the gold doc entirely* (bounded-recall failure that no reranker can fix). *Source:* 2024. + +**RRF / retrieval fusion.** *Mechanism:* combine multiple ranked lists by summing 1/(k+rank), score-agnostic. *Evidence:* Cormack, Clarke & Buettcher, SIGIR 2009 — RRF outperforms Condorcet and learned rank fusion; now the standard hybrid (BM25 + vector) merge. *Limitation:* purely rank-based; if all input lists miss the gold doc, fusion cannot recover it. *Source:* Cormack et al., 2009. **This is a candidate-generation/fusion mechanism, and its evidence is the most robust in the section — but it is not "LLM query transformation."** + +--- + +## 3. Blind-spot pass — challenging the brief + +### 3.1 Attribution: representation vs candidate-generation vs reranking vs "just more compute" + +The brief treats A and B as two "memory-lifecycle interventions." The more useful decomposition of *why any RAG number moves* is four independent levers: + +- **(a) Better representation** — what is stored/embedded (Intervention A; Contextual Retrieval; propositions). +- **(b) Better candidate generation** — how many/which candidates enter the pool (multi-query, HyDE, hybrid+RRF, self-query filters). +- **(c) Better reranking** — reordering a fixed pool (cross-encoder rerankers). +- **(d) Simply spending more inference compute** — any extra LLM call at store or query time. + +Contextual Retrieval isolates (a) cleanly and it is the best-evidenced. RRF isolates fusion within (b). A cross-encoder reranker isolates (c) and, per Anthropic's own numbers, reranking alone drove a large chunk of the 49%→67% improvement — i.e. much of what people attribute to fancy formation is actually **reranking (c)** doing the work. + +**Which intervention is most likely a dressed-up "more compute" effect? Intervention B (query transformation).** Query rewrite/expansion/decomposition each add LLM calls at recall time whose measured "lift" is easily confounded with (d). The "Not All Queries Need Rewriting" and "When Query Expansion Hurts RAG" evidence shows the *sign* of the effect is query-dependent — a hallmark of a weak, compute-driven intervention rather than a structural improvement. If we A/B B without a **latency- and call-count-matched control** (e.g., simple hybrid BM25+vector+RRF+reranker, no LLM rewrite), we will almost certainly misattribute a reranker/compute win to "smart queries." + +Intervention A is *less* likely to be pure-compute, because the CogCanvas and Contextual-Retrieval ablations hold the pipeline fixed and vary only the stored representation. But A has its own trap: the CogCanvas result says the naive version of A (extract structured artifacts to *replace* raw) is *negative*. The only representation change with clean positive isolation is Contextual Retrieval's *additive context prepend on raw chunks* — which is closer to "augment raw" than to "the LLM synthesizes a canonical fact." + +### 3.2 Failure modes the brief under-weights + +- **Memory poisoning / persistent compromise.** Store-time LLM formation and recall-time transformation both widen the attack surface. PoisonedRAG (Zou et al., 2024) achieves >90% attack success with ~5 injected texts/question; AgentPoison (NeurIPS 2024) shows poisoned entries in *long-term memory* persist across sessions and users. An LLM that rewrites memories at store time can *launder* an injection into an authoritative-looking "canonical fact." The brief lists provenance/confidence as *features*; they are also *mitigations that must be load-bearing*, not metadata. +- **Hallucinated facts injected at formation.** This is the highest-severity A-specific risk. An extractor that emits a confident wrong "canonical fact" is worse than storing raw, because raw is at least faithful. Our own mnemon finding that *incomplete* memories can harm output is a direct empirical warning that low-fidelity formation degrades outputs. +- **Query drift / intent distortion.** Documented and quantified: expansion harm rates ~23–42% depending on aggressiveness; harm correlates with lost lexical alignment. B can silently answer a *different* question than the user asked. +- **Latency & cost.** A adds a synchronous (or queued) LLM call per write — at Engram's stated ~183k–344k memory scale this is a real ingestion-throughput and cost line, and our memory notes already record reembed/ingest pipelines saturating. B adds one-to-N LLM calls on the *hot recall path*, directly hitting TTFT. +- **Privacy leakage.** Sending memory contents (A) or user queries (B) to an LLM for reformulation is a new data-egress path; if that LLM is external, it is a compliance surface the brief does not mention. +- **Non-determinism / stability.** Both interventions inject LLM stochasticity into what should be a stable index. Our own out-of-range grader scores (1.10/1.25) and cross-run rank instability already show how fragile measurement is; adding non-deterministic formation/rewrite makes regression detection and reproducibility harder. Formation non-determinism is *permanent* (baked into the stored record); query non-determinism is at least per-request. + +### 3.3 What's missing entirely + +- **Update / conflict / invalidation semantics.** The brief is store-and-recall. Real memory *changes* — facts get superseded (our own memory files are full of "SUPERSEDED" notes). Mem0's ADD/UPDATE/DELETE and Zep's bi-temporal invalidation exist precisely for this. Without it, LLM formation just manufactures more contradictory records faster. +- **Deduplication as a first-class control.** Our own data says near-duplicates *halved* the benefit. That makes dedup/canonicalization a higher-ROI formation-time lever than richer per-record representation — and it is barely implied by the brief. +- **Evaluation harness for the interventions themselves.** mnemon measures memory *value*, not A or B. We need call-count/latency-matched controls, and end-to-end task metrics (not just retrieval recall), before committing. +- **The recall-quality ceiling.** Our binding constraint is ranking under noise. Neither A nor B is the most direct fix — a **cross-encoder reranker + hybrid BM25/vector + RRF** is, and it has the strongest, most independent evidence. + +--- + +## 4. Promising vs. fashionable-but-unsupported + +**Real evidence, worth building (in priority order):** + +1. **Reranking + hybrid (BM25+vector) + RRF.** Not one of the two proposed interventions, but the most robustly evidenced fix for our actual bottleneck (rank-1 collapsing under distractors). RRF: Cormack 2009. Reranking's large marginal effect: Anthropic 2024. **Do this first.** +2. **Contextual Retrieval (additive context prepend on raw chunks).** Best-evidenced *formation* technique (Anthropic 2024, clean isolation, 35–49% failure reduction). It is a conservative form of Intervention A: it *augments* raw, does not replace it. +3. **Deduplication / canonicalization at store time.** Directly targets our own strongest negative finding (duplicates halved the benefit). Cheap, deterministic, high-ROI. +4. **Union storage (raw ∪ derived), never derived-only.** CogCanvas (2026) shows replace-raw is a *negative* result; union matched verbatim. If we do A, do it additively. +5. **HyDE / step-back — situationally.** HyDE helps most where a *tuned* domain retriever is absent (Gao 2022/23); step-back helps *reasoning-heavy multi-hop* (Zheng 2023). Deploy behind a router, not globally. + +**Fashionable but weakly/negatively supported for our case:** + +- **Replacing raw memory with LLM-extracted "canonical facts"/typed artifacts** — the headline framing of Intervention A. Closest controlled evidence (CogCanvas) is *negative*: extraction is lossy distillation and alone never beat naive RAG. High hallucination/poisoning risk. **Do not ship as a replacement.** +- **Store-time "predicted queries" (HyDE-in-reverse) as a primary index** — I could not verify clean published evidence isolating this for open-ended agent memory. **Unverified; prototype-and-measure only.** +- **Global LLM query rewrite/expansion (Intervention B) applied unconditionally** — evidence is explicitly mixed and sign-flips per query (23–42% harm rates; "Not All Queries Need Rewriting," 2026). Most likely to be a **dressed-up "more compute" effect**. If pursued, gate it (selective/router-based) and always test against a compute-matched reranker baseline. +- **Full temporal knowledge graphs (Zep/Graphiti, Mem0g) as the near-term bet** — real and interesting, but the supporting benchmarks are largely vendor-run and near-saturated (DMR 94.8 vs 93.4), and graph construction adds cost/error propagation. Adopt the *idea* we're missing (update/invalidation semantics) before adopting the heavyweight graph. + +**Bottom line for the architecture decision:** Our own numbers say the problem is *ranking under noise* and *corpus hygiene*, not representation poverty. The highest-confidence moves (reranker+hybrid+RRF, dedup, additive contextual embeddings) are largely *outside* the two proposed interventions or are the *conservative additive* form of A. Intervention A as literally framed (replace raw with LLM-synthesized canonical facts) runs against the best controlled evidence; Intervention B is the one most at risk of being a compute mirage and should only be tested selectively against a compute-matched control. Verify every vendor number and any claim marked "unverified" above before it enters a design doc. + +--- + +## Sources + +- Anthropic, "Contextual Retrieval," Sep 19 2024 — https://www.anthropic.com/engineering/contextual-retrieval +- Sarthi et al., "RAPTOR," ICLR 2024 — https://openreview.net/forum?id=GN921JHCRw +- Gao et al., "Precise Zero-Shot Dense Retrieval without Relevance Labels" (HyDE), arXiv:2212.10496, Dec 2022 / ACL 2023 — https://arxiv.org/abs/2212.10496 +- Chen et al., "Dense X Retrieval: What Retrieval Granularity Should We Use?", arXiv:2312.06648, Dec 2023 — https://arxiv.org/abs/2312.06648 +- Chhikara et al., "Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory," arXiv:2504.19413, ECAI 2025 — https://arxiv.org/abs/2504.19413 +- Rasmussen et al., "Zep: A Temporal Knowledge Graph Architecture for Agent Memory," arXiv:2501.13956, Jan 2025 — https://arxiv.org/abs/2501.13956 +- Packer et al., "MemGPT: Towards LLMs as Operating Systems," arXiv:2310.08560, Oct 2023 — https://arxiv.org/abs/2310.08560 +- Zheng et al. (Google DeepMind), "Take a Step Back: Evoking Reasoning via Abstraction in LLMs," arXiv:2310.06117, Oct 2023 — https://arxiv.org/abs/2310.06117 +- Cormack, Clarke, Buettcher, "Reciprocal Rank Fusion...," SIGIR 2009 — https://dl.acm.org/doi/10.1145/1571941.1572114 +- Raudaschl, "RAG-Fusion" — https://github.com/Raudaschl/rag-fusion +- MultiHop-RAG, arXiv:2401.15391 — https://arxiv.org/abs/2401.15391 +- "Question Decomposition for RAG," arXiv:2507.00355 — https://arxiv.org/abs/2507.00355 +- "When Should Queries Be Decomposed?", arXiv:2606.08577, 2026 — https://arxiv.org/abs/2606.08577 +- "Not All Queries Need Rewriting...", arXiv:2603.13301, 2026 — https://arxiv.org/html/2603.13301 +- Multi-Meta-RAG (self-query metadata filtering), Springer 2024 — https://link.springer.com/chapter/10.1007/978-3-031-81372-6_25 +- An, "Verbatim Chunks Beat Extracted Artifacts / CogCanvas," arXiv:2601.00821, 2026 — https://arxiv.org/abs/2601.00821 +- Zou et al., "PoisonedRAG," 2024 — https://arxiv.org/abs/2402.07867 +- Chen et al., "AgentPoison," NeurIPS 2024 — https://proceedings.neurips.cc/paper_files/paper/2024/file/eb113910e9c3f6242541c1652e30dfd6-Paper-Conference.pdf +- mnemon benchmark (internal), `/Users/beauxwalton/projects/mnemon/RESULTS.md` + `reports/*.json` + +*Note on dating: several arXiv IDs above carry 2026-style identifiers (e.g., 2601.*, 2603.*, 2605.*) as surfaced by search in Aug 2026. Where a claim rests on one of these very recent, not-yet-widely-cited preprints (CogCanvas, the query-rewrite-harm papers), I have flagged it as recent/single-source rather than settled.* diff --git a/docs/research/memory-formation-query-transform/03-experiment-spec.md b/docs/research/memory-formation-query-transform/03-experiment-spec.md new file mode 100644 index 00000000..9f56f30b --- /dev/null +++ b/docs/research/memory-formation-query-transform/03-experiment-spec.md @@ -0,0 +1,207 @@ +# Experiment Spec: LLM-Assisted Memory Formation & Query Transformation + +**Status:** planning only — no experiments run yet. +**Base repo:** `/Users/beauxwalton/projects/engram` (branch `staging`, no `main`). +**Downstream harness:** `/Users/beauxwalton/projects/mnemon` (v0.2). +**Date:** 2026-08-23 + +--- + +## 0. Question & interventions + +- **B — LLM-assisted memory formation:** at store time, an LLM rewrites each raw + observation into a retrieval-optimized memory that **preserves exact technical + constraints** (types, units, arg order, key formats, async behavior, + sentinel/return semantics) before embedding. Stored as union (raw ∪ formed). +- **C — LLM-assisted query transformation:** at recall time, an LLM + rewrites/expands/decomposes the query into sub-queries before retrieval. + Engram already exposes `multiQuery` metadata + per-memory `explanations` in + `StructuredQueryResult` (`response_format=structured`) — C instruments this + existing seam rather than inventing infra. + +Evaluated as an **A–E ablation matrix** against retrieval-quality metrics *and* +mnemon's downstream executable-code lift, on **graduated adversarial corpora**. + +--- + +## 1. Datasets / corpora + +Never hard-code mnemon's seeded wording. Each condition is **generated +programmatically** from first-party fact templates (project name, symbol names, +units, key formats), disjoint from `tasks/seed-manifest.ts`. mnemon's 20 target +memories stay the ground-truth positives; all variation lives in the +distractor/perturbation pool seeded into an isolated `ENGRAM_USER_ID` +(`abl--`). Mirrors the `scripts/noisy-corpus.ts` receipt pattern +(per-item write, receipt JSON, targeted cleanup) so every seeded ID is deletable. + +| # | Condition | ~Memories | Construction | +|---|---|---|---| +| G0 | Clean | 20 + 0 | Only the 20 targets. Ceiling baseline. | +| G1 | Ordinary-unrelated noise | 20 + 400 | Off-topic facts from unrelated synthetic projects; distinct vocabulary. | +| G2 | Near-duplicate noise | 20 + 200 | 10 near-topic distractors/target, perturbed contract (reuse `distractor()` shape, template-generated). | +| G3 | Stale / superseded | 20 + 200 | 2–5 older versions/target, `source=HISTORICAL` + earlier `observedAt`, superseded by target. | +| G4 | Directly-contradictory | 20 + 100 | 1–3 opposite-contract memories/target, equal apparent authority. | +| G5 | Ambiguous cross-project | 20 + 200 | Same symbols across 3 fictional projects, different contracts; one is current. | +| F1 | Formation-quality (raw obs) | 20 raw + noise | Positives seeded as verbose raw observations burying the exact constraint in prose a naive summarizer would drop. **The condition B is designed to win.** | + +**Sizing vs saturation:** largest corpus ≈ 420 memories/user. Serialized +single-POST ingestion keeps us far under the ~3.5k-parallel-chunk ceiling that +saturates `engram-embed`. See §5. + +--- + +## 2. Ablation matrix (A–E) + +Storage path and query path are orthogonal switches. B/C are **harness-owned +pre-processing shims** (call an LLM, then hit the same endpoints), so A is +byte-for-byte today's behavior. + +| Cell | Storage | Query | Flags | +|---|---|---|---| +| **A (baseline)** | `POST /v1/memories`, `raw` verbatim, `layer=PROJECT` | original query → `POST /v1/memories/query?scope=user` | `formation=off`, `queryTransform=off`, `--recall-top-k 10` | +| **B** | LLM-formed: `content`=formed text, `raw` retained as provenance | original query | `formation=on`, `queryTransform=off` | +| **C** | current | transformed: {rewrite, expansions, decomposed}; issue each, RRF-fuse client-side, reapply top-k | `formation=off`, `queryTransform=on`, `fusion=rrf` | +| **D (both)** | LLM-formed | transformed | both on, `fusion=rrf` | +| **E (both + rerank)** | LLM-formed | transformed + rerank/context-composition over fused set (cross-encoder or LLM-listwise; dedup; contradiction/stale demotion) | all on, `rerank=on`. **Run only if D shows a positive CI-excludes-zero retrieval gain.** | + +Every cell runs across G0–G5 + F1 and both models (Qwen 3.8 27B, Gemma 4 31B). + +--- + +## 3. Retrieval metrics (independent of generation) + +Computed from the recall response. Ground truth = mnemon's `expectedMemoryIds` +(the `allowed:true` items). Use `response_format=structured` to capture +`recallId`, `multiQuery`, `explanations`; capture `X-Query-Id` header. Harness +reads `items[]` (`id`, `score`, `source`, `content`, `allowed`) + mnemon's +`recall.precision@k`. + +- **hit@k / recall@k** (k∈{1,3,5,10}) +- **MRR** — mean 1/rank of first allowed target +- **nDCG@k** — binary gain, rank-discounted, ideal-normalized +- **precision@k** — mnemon already emits; null when no allowlist (exclude from means) +- **False-positive context-injection rate** (G3/G4/G5) — fraction of selected + items that are stale/contradictory/wrong-project distractors. **The harm metric.** +- **Stability** — issue each query N=10× (fixed seed, no interleaved writes); + report Jaccard of ID sets, rank-set variance, score variance +- **Score diagnostics** — `score` distribution, top1−top2 margin, **tie count** + (the noisy report shows many `score:1.1` ties — real, must track) +- **Latency** — `latencyMs` envelope + wall-clock incl. transform call (C/D/E) +- **Cost** — transform + formation + rerank tokens × price + +Retrieval is scored on mnemon's frozen recall snapshot without spending +generation calls. + +--- + +## 4. Downstream metrics (mnemon) + +**Run command** (from `package.json scripts.mnemon` / README): + +```bash +export ENGRAM_API_KEY=... ENGRAM_USER_ID=abl-- ENGRAM_BASE_URL=http://localhost:3001 +node dist/index.js run \ + --provider openai-compatible --provider-auth optional \ + --base-url http://127.0.0.1:11435/v1 \ + --model omlx/mlx-community--Qwen3.8-27B-8bit \ # or Gemma 4 31B id + --thinking off --backend engram \ + --tasks .mnemon/-tasks --recall-top-k 10 \ + --trials 5 --seed abl---s1 \ + --bootstrap-samples 10000 --output json > report.json +``` + +C/D/E override recall via the transform shim; B/D/E re-seed formed memories first. + +- **Paired executable-quality lift** — mnemon macro-mean overall delta + (memory − no-memory), seeded 10k hierarchical bootstrap 95% CI. + Overall = `0.25·convention + 0.60·executable + 0.15·securityReliability`. +- **Lift conditioned on retrieval hit vs miss** — partition by whether the + allowed target was selected; report lift per stratum. +- **Regression metrics** — on G3/G4/G5, does memory *hurt*? Report memory-arm < + no-memory-arm, correlate with false-positive injection. +- **Cross-model** — every cell × {Qwen, Gemma}. Gemma independently reproducing a + B/C effect is the generalization bar RESULTS.md set. + +--- + +## 5. Reproducibility controls + +- **Seeds:** ≥3 preregistered run seeds/cell; corpus generation is seeded + (deterministic templates → deterministic distractor text/IDs in a receipt). +- **Pinned versions:** fixed Qwen/Gemma IDs, oMLX build, `temperature=0.2`, + `--thinking off`, digest-pinned gate image, mnemon commit, Engram `staging` + commit. Formation/transform/rerank use a fixed LLM + **versioned prompt string** + (`promptVersion`) recorded per record. +- **Embedding-provider pinning:** record per run. Local `engram-embed` only for + small corpora (≤~420 mems) and **serialized ingestion**. Larger → **OpenAI + embeddings** (documented workaround). +- **Saturation guardrail (mandatory pre-run gate):** never bulk/parallel-ingest + graduated corpora against local `engram-embed` — it saturates ~3.5k parallel + chunks, emits malformed sparse vectors pgvector rejects (`22P02`), leaving no + embedding → empty recall (tanked a prior LongMemEval to 1/10). Controls: + (a) serialize writes one-POST-at-a-time; + (b) after seeding, poll `GET /v1/memories/embedding-status` and **abort if any + seeded memory lacks an embedding** (call `POST /v1/memories/embedding-retry`, + re-verify); + (c) prefer OpenAI embeddings above the ceiling. + **No experiment starts until embedding-status is 100% green for that user.** +- **Provenance capture (per query, run-scoped JSONL):** original query; + transformed queries + sub-queries; `layers`/filters/`agentBoost`; candidate set + per sub-query w/ `score`/`source`; fusion strategy + RRF params; rerank + decisions (kept/demoted/deduped + reason); selected `context` + `snapshotHash`; + `recallId` + `X-Query-Id`; formation input→output pairs; model + `promptVersion`; + latency; token cost. + +--- + +## 6. Success thresholds (finalized after baseline) + +Thresholds are set **after** cell A establishes baseline distribution + CI width. +Illustrative framing: + +- **Ship B** if, on F1 + G2–G5, B beats A on retrieval hit@5 **and** downstream + lift with 95% bootstrap CI excluding zero, on **both** models, with **no** + regression on G0. +- **Ship C** if C beats A on MRR/nDCG@5 + downstream lift, CI excluding zero, + without raising false-positive injection on G3/G4/G5 beyond a set tolerance. +- **Ship D over B or C** only if D's lift exceeds the better single intervention + by a margin whose CI excludes zero (guards against additive-cost-no-benefit). +- **Ship E** only if it reduces false-positive injection on G3/G4/G5 *and* + preserves G0/F1 lift. +- A single positive seed is never sufficient — claims require CI-excludes-zero + across preregistered seeds. + +--- + +## 7. Cost & latency budgets + +- **Generation calls/cell:** `valid_tasks × trials × 2` = 20×5×2 = 200/run. Full + matrix ≈ 5 cells × 7 corpora × 2 models × 3 seeds ≈ 210 runs ≈ 42k generation + calls — local oMLX, compute-bound not $-bound. `--gate-timeout 60000`, + `--model-timeout 120000`. +- **Intervention LLM overhead:** formation = one call/seeded memory (~420 max, + one-time/seed). Transform = one call/task-query (×20). Rerank (E) = one + call/query over ≤k candidates. Hard cost cap/full-matrix on a small hosted + model, recorded per record; abort a cell if projected cost exceeds cap. +- **Latency budget:** added recall latency ≤ transform p95 + fusion (target <2s/q + for C; <4s for E). Report recall `latencyMs` separately from transform + wall-clock. + +--- + +## Implementer notes (from repo inspection) + +- Engram already has query-expansion plumbing: `StructuredQueryResult.multiQuery` + + per-memory `explanations` via `POST /v1/memories/query?response_format=structured`. + C should populate/consume these, not build parallel infra. `X-Query-Id` + + `recallId` give free retrieval-signal correlation. +- Score ties are common (`score:1.1` across many items) and `allowed:false` + distractors get injected into `context` — false-positive-injection and + tie-count metrics are load-bearing, not theoretical. +- `GET /v1/memories/embedding-status` + `POST /v1/memories/embedding-retry` exist + — the concrete hooks for the saturation guardrail. +- `CreateMemoryDto` accepts both `raw` and `content` (+ `source`, `layer`); B + stores formed text in `content`, retains `raw` for provenance — **no schema + change.** +- Run: `node dist/index.js run` (built) / `npm run mnemon -- run` (tsx). + Seed/cleanup: `npm run seed:noisy` / `cleanup:noisy` (`scripts/noisy-corpus.ts`). diff --git a/docs/research/memory-formation-query-transform/README.md b/docs/research/memory-formation-query-transform/README.md new file mode 100644 index 00000000..43ddda1b --- /dev/null +++ b/docs/research/memory-formation-query-transform/README.md @@ -0,0 +1,82 @@ +# Engram R&D: LLM-Assisted Memory Formation & Query Transformation — Phase 1 + +**Owner:** Rook · **Requested by:** Kit (brief) / Beaux (approval) · **Date:** 2026-08-23 +**Phase:** 1 (read-only research). No code, data, or migrations touched. +**Base branch:** `staging` (there is no `main`; staging is newest + our R&D target). + +This directory holds the Phase 1 deliverables for the initiative to evaluate two +interventions in Engram's memory lifecycle: + +- **Formation** — use an LLM at *store* time to produce a more precise, retrievable + memory representation. +- **Query transformation** — use an LLM at *recall* time to rewrite/expand/decompose + the query before retrieval. + +## Documents + +| # | File | What it is | +|---|---|---| +| 1 | [`01-current-state-map.md`](01-current-state-map.md) | Ingestion→recall data-flow audit, score semantics, data model, intervention points, existing LLM usage | +| 2 | [`02-research-memo.md`](02-research-memo.md) | Literature review (formation + query-transform), blind-spot pass, promising-vs-hype read, cited primary sources | +| 3 | [`03-experiment-spec.md`](03-experiment-spec.md) | A–E ablation matrix, graduated corpora, retrieval + downstream metrics, reproducibility controls, budgets | + +## Executive summary + +**The mnemon evidence points at ranking-under-noise and corpus hygiene as our +binding constraint — not representation poverty.** Rank-1 retrieval collapsed +20/20 → 12–13/20 once distractors were added; the biggest observed harms came +from duplicate and incomplete memories. This shapes which intervention is likely +to pay off. + +**Formation (store-time):** the closest published analogue to "replace raw text +with LLM-extracted structure" is a *negative* result — CogCanvas (arXiv:2601.00821, +2026), a controlled ablation where extracted artifacts **never beat naive RAG** and +lost 15.9–22 pts to verbatim chunks; the only design that matched verbatim was +**raw ∪ derived (union)**. Anthropic's **Contextual Retrieval** (Sep 2024) is the +best-evidenced *additive* formation win (35–67% retrieval-failure reductions). +**Implication: pursue formation as union storage + a contextualizing prepend, +never as raw-replacement.** Engram's schema already supports this (retain `raw`, +fill `content`) with no migration. + +**Query transformation (recall-time):** the intervention most at risk of being a +dressed-up "more inference compute" effect. Published evidence shows the *sign* +of the effect flips per-query (rewriting helps some, harms 23–42% of others). It +**must** be tested against a latency/call-count-matched reranker baseline, not +just against A. Engram already exposes `multiQuery` + `explanations` plumbing to +instrument this without new infra. + +**Correctness smell found during the audit (independent of the R&D):** recall +scores exceed [0,1] because keyword-rescue hits are hard-coded to **1.25** (FTS) +and **1.1** (ILIKE) and graph hits get **×1.2**. This is why wrong distractors +tie correct memories at 1.25. Scores are relative sort ranks, not probabilities. +Recommend a ticket to document/calibrate this regardless of the initiative. + +## Recommended sequencing (from the research) + +1. **Baseline (cell A)** on graduated corpora — establish CI width before thresholds. +2. **Hybrid + rerank + RRF tuning first** — it targets our actual bottleneck and is + the best-evidenced lever; also the fair baseline the query-transform arm must beat. +3. **Formation as union + contextual prepend** (cell B) on the F1 corpus it's + designed to win. +4. **Query transform** (cell C) **only vs a compute-matched reranker baseline.** +5. Cells D/E only if D shows a CI-excludes-zero retrieval gain. + +## Hard constraints honored + +- No production deploy, no destructive migrations, no `prisma migrate dev`. +- Raw memories + API compatibility preserved (union storage, additive metadata). +- All prototypes will be feature-flagged and reversible. +- Large-corpus experiments route around the **`engram-embed` saturation bug** + (serialized ingestion + `embedding-status` gate + OpenAI-embedding fallback). + +## Open questions for Beaux (only the ones that change architecture/cost) + +1. **Formation scope:** OK to constrain formation to **union storage + contextual + prepend** (evidence-backed) and explicitly *rule out* raw-replacement and + store-time "predicted queries" (unverified) for the prototype? +2. **Query-transform fairness bar:** agree that C must beat a **compute-matched + reranker baseline**, not just cell A, before it earns complexity? +3. **Prototype green-light:** the brief authorizes proceeding into the smallest + feature-flagged vertical slice after Phase 1. Want me to proceed to that + (instrumentation + baseline run on the small/safe corpora), or hold for your + review of these docs first? From 3540c2cbf0f1926b21e9b92b97201a2b628b7df0 Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 12:08:56 -0700 Subject: [PATCH 2/8] docs(research): correct Phase 1 errata found during prototype build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three Phase 1 assumptions were falsified by running against live Engram: - Local Engram serves on 47291, not 3001 (3001 is whalehawk provider-server) - `content` is a legacy alias for `raw`, not a second field. The documented "store formed text in content, keep raw for provenance" plan would have been a silent no-op and invalidated intervention B. Union is now realized inside the embedded string via contextual prefix + verbatim observation with recorded prefixLength. - Local embedder 500s under parallel recall, which silently degraded arm C into the baseline — a bias against the intervention under test Also documents a pre-existing pgvector dimension mismatch (1536 vs 384) on the write path, recommended for its own ticket. Co-Authored-By: Claude Opus 4 --- .../01-current-state-map.md | 59 +++++++++++++++++-- .../README.md | 15 +++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/docs/research/memory-formation-query-transform/01-current-state-map.md b/docs/research/memory-formation-query-transform/01-current-state-map.md index b2e4d667..31ba95cc 100644 --- a/docs/research/memory-formation-query-transform/01-current-state-map.md +++ b/docs/research/memory-formation-query-transform/01-current-state-map.md @@ -1,7 +1,17 @@ # Current-State Map: Engram Memory Lifecycle (Ingestion → Recall) **Scope:** Read-only audit of Engram's `staging` branch. No code or data modified. -**Date:** 2026-08-23 +**Date:** 2026-08-23 (revised same day after prototype-build verification) + +> **Errata from the prototype build.** Three assumptions in the original revision +> were falsified by running against the live service. All are corrected inline: +> 1. **Local Engram serves on port `47291`, not `3001`.** Port 3001 is the +> whalehawk provider-server (returns `Hello World!` at `/`, 404s on +> `/v1/memories/*`). Verified: `47291` returns 401 on +> `/v1/memories/embedding-status`; `3001` returns 404. +> 2. **`content` is a legacy alias for `raw`, not a second field** — see §4. +> 3. **The local embedder 500s under parallel recall** (`non-finite value +> NaN/inf` from minilm) — see §7. **Purpose:** Map the ingestion→recall data flow and identify intervention points for (A) LLM-assisted memory formation at store time and (B) LLM-assisted query transformation at recall time. @@ -155,11 +165,26 @@ transformation at recall time. - `MemoryEventTime` — resolved temporal references ### What formation would add -- Store formed/canonical text in `content` (DTO already accepts `content` + - `raw`), retaining `raw` for provenance — **no schema change required for a - union-storage MVP.** -- Optional: `metadata.formation = {formationScore, promptVersion, model, - suggestedRaw}` or a dedicated `MemoryFormation` table if we want it queryable. + +> **CORRECTION (2026-08-23, verified against source during prototype build).** +> An earlier revision of this document claimed `CreateMemoryDto` accepts `content` +> and `raw` as two independent fields, and that formed text could go in `content` +> with `raw` kept for provenance. **That is false.** `create-memory.dto.ts` (~L94) +> applies `@Transform(({ value, obj }) => value ?? obj.content)` to `raw`, and the +> adjacent comment reads *"Legacy alias: content -> raw (accepted but transformed +> to raw)"*. There is **no `content` column on `Memory`** — posting both discards +> `content`. Storing formed text in `content` would have been a **silent no-op** +> and would have invalidated intervention B. + +- **Union must be realized inside the single embedded string.** The prototype + stores `contextualPrefix + verbatimObservation` in `raw`, and records + `prefixLength` in metadata so the original observation is recoverable + **byte-for-byte** (`storedText.slice(prefixLength) === observation`, asserted at + write time). This is precisely Anthropic's Contextual Retrieval shape and it + preserves the raw-∪-derived property the CogCanvas result demands. +- Provenance rides in `metadata.formation = {promptVersion, model, prefixLength, + constraintEchoes[]}`. A dedicated `MemoryFormation` table is the option if we + later want it queryable — not needed for the MVP. --- @@ -198,6 +223,28 @@ a clean feature-flag precedent already exists. --- +## 7. Infrastructure defects found during the prototype build + +Both were surfaced by running the shims against live Engram + oMLX. Both are +**measurement hazards** — left unhandled they bias experimental results. + +### 7.1 Local embedder 500s under parallel recall (biases intervention C downward) +Issuing sub-queries concurrently (which is exactly what query expansion does) +made the local embedder return 500 `non-finite value NaN/inf` from minilm. In the +first smoke run this forced arm C to fall back to baseline recall — i.e. **the +intervention silently degraded into the control**, which would have understated C +in the ablation. Mitigated in the harness with bounded retries + sub-query +concurrency limiting (second run: zero fall-backs). Any future arm that fans out +queries must keep this limiter or the comparison is invalid. + +### 7.2 pgvector dimension mismatch on write (pre-existing, unrelated to this R&D) +Engram logs a dimension mismatch (**1536 vs 384**) on the write path — an +openai-small-shaped vector meeting a minilm-shaped column. This is independent of +the shims and predates them. It is the same *class* of failure as the known +`engram-embed` saturation bug (writes that silently end up with no usable +embedding → empty recall). **Recommend a ticket**; until resolved, pin and record +the embedder per run, and keep the `embedding-status` gate mandatory. + ## Key implications for the R&D - The binding constraint the mnemon data exposes is **ranking under noise** and diff --git a/docs/research/memory-formation-query-transform/README.md b/docs/research/memory-formation-query-transform/README.md index 43ddda1b..cec3aa67 100644 --- a/docs/research/memory-formation-query-transform/README.md +++ b/docs/research/memory-formation-query-transform/README.md @@ -45,6 +45,21 @@ of the effect flips per-query (rewriting helps some, harms 23–42% of others). just against A. Engram already exposes `multiQuery` + `explanations` plumbing to instrument this without new infra. +**Findings from the prototype build that corrected this research** (see +`01-current-state-map.md` errata + §7): +- Local Engram is on port **47291**, not 3001 (3001 is whalehawk provider-server). +- **`content` is a legacy alias for `raw`, not a second field.** The original plan + to "store formed text in `content`, keep `raw` for provenance" would have been a + **silent no-op**. Union is instead realized inside the single embedded string + (contextual prefix + verbatim observation, `prefixLength` recorded so the + original is recoverable byte-for-byte) — which is Anthropic's Contextual + Retrieval shape and still satisfies raw ∪ derived. +- The local embedder **500s under parallel recall**, which silently degraded arm C + into the baseline on the first smoke run — a bias against the intervention we + are trying to measure. Now mitigated with concurrency limiting. +- A **pre-existing pgvector dimension mismatch (1536 vs 384)** is logged on write. + Unrelated to this work; recommend a separate ticket. + **Correctness smell found during the audit (independent of the R&D):** recall scores exceed [0,1] because keyword-rescue hits are hard-coded to **1.25** (FTS) and **1.1** (ILIKE) and graph hits get **×1.2**. This is why wrong distractors From 7896a452941c4b2815bf0f1e503e3ad65473804f Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 12:18:12 -0700 Subject: [PATCH 3/8] docs(research): retrieval ranking is tie-dominated, not score-decided Verified independently twice against the committed v0.2 noisy report: 18/20 queries end in a 5-or-more-way tie at the top score, and only 2/20 top-1 results are decided by score. The reported "13/20 rank 1" is 11-of-13 decided by array order within a tied block; under fair tie-breaking the same system scores ~0.21, not 0.65. Context false-positive injection rate is 0.87. Root cause is the score semantics already documented in the current-state map: keyword-rescue paths assign hard-coded constants (FTS 1.25, ILIKE 1.1) rather than a similarity, so every rescued candidate lands on an identical score. This blocks the A-E ablation: interventions work by supplying better candidates, but a better candidate cannot change top-1 when the tie-break decides it. Sequencing revised so scoring/tie-breaking is fixed before the matrix runs. Downstream lift figures in RESULTS.md are unaffected; the retrieval narrative attached to them is not. Co-Authored-By: Claude Opus 4 --- .../04-finding-tie-domination.md | 122 ++++++++++++++++++ .../README.md | 21 ++- 2 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 docs/research/memory-formation-query-transform/04-finding-tie-domination.md diff --git a/docs/research/memory-formation-query-transform/04-finding-tie-domination.md b/docs/research/memory-formation-query-transform/04-finding-tie-domination.md new file mode 100644 index 00000000..addd772c --- /dev/null +++ b/docs/research/memory-formation-query-transform/04-finding-tie-domination.md @@ -0,0 +1,122 @@ +# Finding: Retrieval Ranking Is Tie-Dominated, Not Score-Decided + +**Date:** 2026-08-23 +**Status:** verified independently (twice, by separate analyses of the same +committed report) +**Severity:** blocks meaningful evaluation of both interventions +**Source data:** `mnemon/reports/mnemon-v0.2-qwen-engram-noisy-20task-3x-t02.json` + +--- + +## The finding + +On the 20-task noisy corpus, Engram's top-1 retrieval result is almost never +determined by the score. It is determined by **array order among tied scores.** + +| Measure | Value | +|---|---| +| Unique task-snapshots analyzed | 20 | +| Snapshots with a **5-or-more-way tie at the top score** | **18 / 20** | +| Snapshots where top-1 was **uniquely decided by score** | **2 / 20** | +| `hit@1` as returned (matches `RESULTS.md` "13/20 rank 1") | 13 / 20 = **0.65** | +| `hit@1` expected under fair tie-breaking | **≈ 0.21** | +| `hit@1` pessimistic | **≈ 0.10** | +| Context false-positive injection rate | **0.87** | + +**11 of the 13 "rank 1" successes were not earned by ranking.** They are the +consequence of the correct memory happening to sit earlier in a tied block. Under +random tie-breaking the same system scores ~0.21; under adversarial ordering, +~0.10. + +Separately: **87% of items injected into the model's context are distractors** +(`allowed: false`). + +## Why this happens + +This traces directly to the score semantics documented in +[`01-current-state-map.md` §3](01-current-state-map.md). Engram's recall score is +not a similarity measure end-to-end: + +- cosine path → `1 - distance`, genuinely in [0,1] and continuous +- **full-text-search rescue → hard-coded `1.25`** +- **ILIKE rescue → hard-coded `1.1`** +- graph-recall hits → `× 1.2` + +The rescue paths assign **constants**. Every memory that qualifies via keyword +rescue receives *the identical score*, regardless of how well it matches. In a +noisy corpus, many near-duplicate distractors qualify — so they all land on 1.25 +together with the correct memory, and the tie is broken by whatever order the +rows came back in. + +The scoring function has **almost no discriminative power in exactly the regime we +care about.** + +## Why this blocks the study + +The ablation matrix (A–E) measures whether better *representation* (formation, B) +or better *candidate generation* (query transformation, C) improves retrieval. +Both interventions work by putting better candidates into the result set. + +**If 18/20 queries end in a 5-way tie at the top, a better candidate does not +change top-1 — the tie-break does.** Any measured improvement or regression would +be dominated by ordering noise rather than by the intervention. We would be +measuring the row order of a Postgres result set. + +Concretely, this invalidates: +- `hit@1` and MRR as primary endpoints (both are tie-sensitive) +- any A-vs-B or A-vs-C comparison whose delta is smaller than the tie envelope +- the previously reported rank-1 counts as a baseline to improve on + +It also explains a prior anomaly we had attributed to index settling: identical +corpora scoring **13/20 one run and 12/20 another**. That is not drift in +retrieval quality; it is the tie order resolving differently. + +## Consequences for prior results + +The **downstream** lift numbers in `RESULTS.md` (+0.219 clean, +0.101/+0.117 +noisy, CIs excluding zero) are **not invalidated** — those are paired +executable-code measurements and stand on their own. What is invalidated is the +*retrieval-quality* narrative attached to them: we reported 13/20 rank-1 as +evidence that retrieval was mostly working under noise. It was not. Retrieval was +producing a 5-way tie and getting lucky. + +If anything this **strengthens** the case that memory helps — the downstream lift +was achieved *despite* an 87% context false-positive rate and essentially +arbitrary top-1 selection. + +## Recommended sequencing change + +The Phase 1 memo already argued (from the CogCanvas and query-rewrite-harm +literature) that reranking should come before either LLM intervention. This +finding makes that ordering **mandatory rather than advisory**: + +1. **Fix score semantics first.** The rescue constants (1.25 / 1.1) and the ×1.2 + graph boost must produce a continuous, discriminative ordering — or ties must + be broken by an explicit, principled secondary key (e.g. cosine similarity) + rather than result-set order. Until then the system cannot express a + preference between candidates. +2. **Re-establish the baseline** with tie-aware metrics. Report the four-way tie + envelope (`expected` / `optimistic` / `pessimistic` / `asReturned`), never a + bare scalar. +3. **Then** run the A–E matrix. Only once ranking is discriminative can formation + or query transformation be credited or blamed for a change in top-k. + +## Measurement requirements going forward + +- **Never report `asReturned` alone.** It flattered our previous results by 3×. + The harness (`ablation/metrics/`) emits the tie envelope by construction. +- **Deduplicate trial copies.** All 3 trials share one `snapshotHash` per task; + treating them as independent narrows every CI by ~√3 for zero added + information. +- **Stability needs pooled runs.** It cannot be measured within a single report, + because the snapshot is frozen per task. +- **Track the tie count and top1−top2 margin as first-class metrics**, not + diagnostics. They are the leading indicator of whether any retrieval delta is + real. + +## Verification + +Confirmed twice by independent paths: once by the metrics implementation against +hand-computed fixtures, and once by a direct recomputation over the raw report +JSON. Both produced identical numbers (18/20 five-way ties, 2/20 score-decided, +13/20 `asReturned`, 0.87 false-positive rate). diff --git a/docs/research/memory-formation-query-transform/README.md b/docs/research/memory-formation-query-transform/README.md index cec3aa67..83de48bc 100644 --- a/docs/research/memory-formation-query-transform/README.md +++ b/docs/research/memory-formation-query-transform/README.md @@ -19,6 +19,7 @@ interventions in Engram's memory lifecycle: | 1 | [`01-current-state-map.md`](01-current-state-map.md) | Ingestion→recall data-flow audit, score semantics, data model, intervention points, existing LLM usage | | 2 | [`02-research-memo.md`](02-research-memo.md) | Literature review (formation + query-transform), blind-spot pass, promising-vs-hype read, cited primary sources | | 3 | [`03-experiment-spec.md`](03-experiment-spec.md) | A–E ablation matrix, graduated corpora, retrieval + downstream metrics, reproducibility controls, budgets | +| 4 | [`04-finding-tie-domination.md`](04-finding-tie-domination.md) | **Blocking finding.** Retrieval ranking is decided by array order among tied scores, not by score. Changes the required sequencing. | ## Executive summary @@ -66,14 +67,24 @@ and **1.1** (ILIKE) and graph hits get **×1.2**. This is why wrong distractors tie correct memories at 1.25. Scores are relative sort ranks, not probabilities. Recommend a ticket to document/calibrate this regardless of the initiative. -## Recommended sequencing (from the research) +## Recommended sequencing (REVISED after the tie-domination finding) -1. **Baseline (cell A)** on graduated corpora — establish CI width before thresholds. -2. **Hybrid + rerank + RRF tuning first** — it targets our actual bottleneck and is - the best-evidenced lever; also the fair baseline the query-transform arm must beat. +Phase 2 measurement work produced a blocking result (see +[`04`](04-finding-tie-domination.md)): **18/20 queries end in a 5-way tie at the +top score, and only 2/20 top-1 results are decided by score at all.** Better +candidates cannot move top-1 when the tie-break decides it, so the A–E matrix +would largely measure Postgres row order. + +Sequencing is therefore now: + +1. **Fix score semantics / tie-breaking first** (was step 2, now mandatory and + first). The hard-coded rescue constants (FTS `1.25`, ILIKE `1.1`) and the + `×1.2` graph boost collapse distinct candidates onto identical scores. +2. **Re-establish baseline (cell A)** with tie-aware metrics — always report the + four-way tie envelope, never `asReturned` alone. 3. **Formation as union + contextual prepend** (cell B) on the F1 corpus it's designed to win. -4. **Query transform** (cell C) **only vs a compute-matched reranker baseline.** +4. **Query transform** (cell C) **only vs the compute-matched control.** 5. Cells D/E only if D shows a CI-excludes-zero retrieval gain. ## Hard constraints honored From c9054387bd71a7c28fbe3e32191419ea1c7d027b Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 13:44:36 -0700 Subject: [PATCH 4/8] fix(recall): make rescue scores continuous and ties deterministic Engram's top-1 recall result was decided by Postgres row order, not by score. On the 20-task noisy benchmark corpus 18/20 queries ended in a 5-or-more-way tie at the top score and only 2/20 were uniquely decided by score, so the published "13/20 rank 1" collapses to ~0.21 under fair tie-breaking. See docs/research/memory-formation-query-transform/04-finding-tie-domination.md. Two independent causes: 1. Both keyword-rescue paths assigned constants. The FTS/BM25 path computed an RRF value and threw it away, calling scoreMap.set(id, 1.25) for every rescued candidate; the ILIKE path did the same with 1.1. The SQL already ordered by ts_rank but never SELECTed it. 2. Every score sort was a bare (b.score ?? 0) - (a.score ?? 0). Because Array.prototype.sort is stable, equal scores kept whatever order the DB returned. Fix, in new src/memory/memory-ranking.util.ts: - ftsRescueScore() maps ts_rank (now SELECTed in all three FTS query variants) into (1.10, 1.25]: 0.9 * (ts_rank / max ts_rank) + 0.1 * normalised RRF. Band top is exactly 1.25 for the best hit and values decrease strictly with rank, so near-duplicates with identical ts_rank no longer collapse into a tie. - ilikeRescueScore() maps lexical coverage (matched terms / extracted terms, computed in SQL) into (1.00, 1.10] with the same RRF tie-break. - compareByRankKeys() is the single comparator now used by every score sort in memory-query.service.ts and memory-query-ranking.service.ts: score desc, vectorScore (cosine) desc, importanceScore desc, createdAt desc, id asc. The final key makes the ordering a total order, so the same candidate set sorts identically regardless of DB row order. - Sticky keyword re-add after reranking preserves the banded score instead of clamping to 1.1. Inter-band priority is unchanged on purpose: FTS rescue still outranks ILIKE rescue, which still outranks every cosine hit (ceiling 1.0). The graph-recall x1.2 boost and cross-encoder reranker behaviour are untouched so the re-baseline measures one variable. Tests: new memory-ranking.util.spec.ts (17) plus a regression block in memory-query.service.spec.ts asserting distinct scores for distinct ts_rank, band ordering vs a 0.99 cosine hit, and identical output for the same tied set fed in three different input orders. src/memory: 66 suites / 1143 tests pass. --- src/memory/memory-query-ranking.service.ts | 8 +- src/memory/memory-query.service.spec.ts | 154 +++++++++++++++ src/memory/memory-query.service.ts | 141 ++++++++++--- src/memory/memory-ranking.util.spec.ts | 217 +++++++++++++++++++++ src/memory/memory-ranking.util.ts | 182 +++++++++++++++++ src/memory/memory.types.ts | 6 + 6 files changed, 674 insertions(+), 34 deletions(-) create mode 100644 src/memory/memory-ranking.util.spec.ts create mode 100644 src/memory/memory-ranking.util.ts diff --git a/src/memory/memory-query-ranking.service.ts b/src/memory/memory-query-ranking.service.ts index a13b00b9..d36947e4 100644 --- a/src/memory/memory-query-ranking.service.ts +++ b/src/memory/memory-query-ranking.service.ts @@ -7,6 +7,7 @@ import { RecallWeightService } from './recall-weight.service'; import { RerankService } from '../embedding/rerank.service'; import { GraphRecallService } from './graph-recall.service'; import { SentimentService } from './sentiment.service'; +import { compareByRankKeys } from './memory-ranking.util'; export interface InsightSurfacingOptions { allow?: boolean; @@ -97,7 +98,7 @@ export class MemoryQueryRankingService { scoredMemories.push(gm); } } - scoredMemories.sort((a, b) => (b.score ?? 0) - (a.score ?? 0)); + scoredMemories.sort(compareByRankKeys); return scoredMemories; } @@ -185,6 +186,7 @@ export class MemoryQueryRankingService { relevantInsights.push({ ...insight, score: boostedScore, + vectorScore: similarity, } as MemoryWithScore); } } @@ -196,7 +198,7 @@ export class MemoryQueryRankingService { // Slicing to `limit` before reranking drops gold memories that the // cross-encoder would correctly promote. const merged = [...existingResults, ...relevantInsights].sort( - (a, b) => (b.score ?? 0) - (a.score ?? 0), + compareByRankKeys, ); this.logger.log( @@ -240,7 +242,7 @@ export class MemoryQueryRankingService { sp; return { ...m, score: finalScore }; }) - .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) + .sort(compareByRankKeys) .slice(0, limit); if (!this.rerankService || memories.length === 0) { diff --git a/src/memory/memory-query.service.spec.ts b/src/memory/memory-query.service.spec.ts index 1ceefd0d..e533ac40 100644 --- a/src/memory/memory-query.service.spec.ts +++ b/src/memory/memory-query.service.spec.ts @@ -533,6 +533,160 @@ describe('MemoryQueryService', () => { }); }); + // Regression suite for the tie-domination defect: + // docs/research/memory-formation-query-transform/04-finding-tie-domination.md + // + // Before this fix every FTS-rescued candidate scored a flat 1.25 and every + // ILIKE-rescued candidate a flat 1.1, and every sort was a bare score + // comparator over a stable sort — so top-1 was decided by Postgres row + // order, not by relevance. + describe('keyword rescue scoring is continuous and deterministic', () => { + const mem = (id: string, overrides: Record = {}) => ({ + id, + raw: `memory ${id}`, + importanceScore: 0.5, + createdAt: new Date('2026-01-01T00:00:00Z'), + extraction: {}, + ...overrides, + }); + + /** + * Routes raw SQL by shape: the FTS query SELECTs ts_rank, the lexical + * rescue query SELECTs match_count. + */ + const mockRawSql = (opts: { + fts?: { id: string; ts_rank: number }[]; + ilike?: { id: string; match_count: number }[]; + }) => + jest.fn().mockImplementation((sql: string) => { + if (sql.includes('match_count')) + return Promise.resolve(opts.ilike ?? []); + if (sql.includes('ts_rank')) return Promise.resolve(opts.fts ?? []); + return Promise.resolve([]); + }); + + it('gives FTS-rescued candidates with different ts_rank different scores', async () => { + embedding.search.mockResolvedValue([]); + (prisma as any).$queryRawUnsafe = mockRawSql({ + fts: [ + { id: 'f1', ts_rank: 0.9 }, + { id: 'f2', ts_rank: 0.1 }, + ], + }); + prisma.memory.findMany = jest + .fn() + .mockResolvedValue([mem('f2'), mem('f1')]); + + const result = await service.recall(userId, { query: 'roast' } as any); + + expect(result.memories).toHaveLength(2); + const [first, second] = result.memories; + expect(first.score).not.toEqual(second.score); + // Higher ts_rank wins, regardless of the order the DB returned rows in. + expect(first.id).toBe('f1'); + expect(second.id).toBe('f2'); + }); + + it('separates FTS-rescued candidates even when ts_rank ties exactly', async () => { + embedding.search.mockResolvedValue([]); + (prisma as any).$queryRawUnsafe = mockRawSql({ + fts: [ + { id: 'f1', ts_rank: 0.5 }, + { id: 'f2', ts_rank: 0.5 }, + { id: 'f3', ts_rank: 0.5 }, + ], + }); + prisma.memory.findMany = jest + .fn() + .mockResolvedValue([mem('f1'), mem('f2'), mem('f3')]); + + const result = await service.recall(userId, { query: 'roast' } as any); + + const scores = result.memories.map((m) => m.score); + expect(new Set(scores).size).toBe(3); + }); + + it('keeps an FTS rescue above a 0.99 cosine hit', async () => { + embedding.search.mockResolvedValue([{ id: 'c1', score: 0.99 }] as any); + (prisma as any).$queryRawUnsafe = mockRawSql({ + fts: [{ id: 'f1', ts_rank: 0.2 }], + }); + prisma.memory.findMany = jest + .fn() + .mockResolvedValue([mem('c1'), mem('f1')]); + + const result = await service.recall(userId, { query: 'roast' } as any); + + expect(result.memories[0].id).toBe('f1'); + expect(result.memories[0].score!).toBeGreaterThan( + result.memories[1].score!, + ); + }); + + it('keeps an FTS rescue above an ILIKE rescue (band ordering preserved)', async () => { + embedding.search.mockResolvedValue([]); + (prisma as any).$queryRawUnsafe = mockRawSql({ + fts: [{ id: 'f1', ts_rank: 0.01 }], + ilike: [{ id: 'l1', match_count: 1 }], + }); + prisma.memory.findMany = jest + .fn() + .mockResolvedValue([mem('l1'), mem('f1')]); + + const result = await service.recall(userId, { query: 'roast' } as any); + + expect(result.memories.map((m) => m.id)).toEqual(['f1', 'l1']); + }); + + it('ranks ILIKE rescues by lexical coverage, not a flat constant', async () => { + embedding.search.mockResolvedValue([]); + (prisma as any).$queryRawUnsafe = mockRawSql({ + // SQL order is importance/createdAt, so the low-coverage row comes + // first — coverage must still promote the 2-of-2 match. + ilike: [ + { id: 'l1', match_count: 1 }, + { id: 'l2', match_count: 2 }, + ], + }); + prisma.memory.findMany = jest + .fn() + .mockResolvedValue([mem('l1'), mem('l2')]); + + const result = await service.recall(userId, { + query: 'roast medication', + } as any); + + expect(result.memories.map((m) => m.id)).toEqual(['l2', 'l1']); + expect(result.memories[0].score).not.toEqual(result.memories[1].score); + }); + + it('resolves genuinely equal scores deterministically, not by input order', async () => { + // Five candidates that tie on cosine score — the exact regime that made + // 18/20 benchmark queries tie-dominated. The output must be identical + // regardless of the order Postgres returned the rows in. + const tied = ['m1', 'm2', 'm3', 'm4', 'm5']; + const runWithOrder = async (order: string[]) => { + embedding.search.mockResolvedValue( + tied.map((id) => ({ id, score: 0.5 })) as any, + ); + (prisma as any).$queryRawUnsafe = mockRawSql({}); + prisma.memory.findMany = jest + .fn() + .mockResolvedValue(order.map((id) => mem(id))); + const result = await service.recall(userId, { query: 'roast' } as any); + return result.memories.map((m) => m.id); + }; + + const forward = await runWithOrder(tied); + const reversed = await runWithOrder([...tied].reverse()); + const shuffled = await runWithOrder(['m3', 'm5', 'm1', 'm4', 'm2']); + + expect(forward).toEqual(reversed); + expect(forward).toEqual(shuffled); + expect(forward).toEqual(tied); + }); + }); + describe('shouldUseMultiQuery', () => { it('should return false when multiQueryService is not available', () => { const recallWeightService = { diff --git a/src/memory/memory-query.service.ts b/src/memory/memory-query.service.ts index ea5952c4..0df91017 100644 --- a/src/memory/memory-query.service.ts +++ b/src/memory/memory-query.service.ts @@ -42,6 +42,12 @@ import { RecallWeightService } from './recall-weight.service'; import { MemoryQueryRankingService } from './memory-query-ranking.service'; import { MemoryQueryContextService } from './memory-query-context.service'; import { MemoryFailureService } from './memory-failure.service'; +import { + compareByRankKeys, + ftsRescueScore, + ilikeRescueScore, + ILIKE_RESCUE_BAND_BOTTOM, +} from './memory-ranking.util'; @Injectable() export class MemoryQueryService { @@ -351,9 +357,13 @@ export class MemoryQueryService { blendedScore * this.recallWeightService.recallWeight(memory) * this.rankingService.getImportanceMultiplier(memory); - return { ...memory, score: adjustedScore } as MemoryWithScore; + return { + ...memory, + score: adjustedScore, + vectorScore: scoreMap.get(memory.id), + } as MemoryWithScore; }) - .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) + .sort(compareByRankKeys) .slice(0, TEMPORAL_RERANK_POOL); // wide pool — reranker will final-sort to `limit` } else { // STANDARD PATH (ENG-26: pass query text for hybrid search fusion) @@ -371,6 +381,10 @@ export class MemoryQueryService { ); const scoreMap = new Map(vectorResults.map((r) => [r.id, r.score])); + // Preserved separately from scoreMap: the keyword-rescue paths below + // overwrite scoreMap with band values, but the raw cosine similarity is + // the first tie-break key (see compareByRankKeys). + const cosineMap = new Map(vectorResults.map((r) => [r.id, r.score])); const memoryIds = vectorResults.map((r) => r.id); const keywordRescueIds = new Set(); @@ -386,9 +400,12 @@ export class MemoryQueryService { ? [singleUserId] : []; try { + type FtsRow = { id: string; ts_rank: number }; const ftsResults = poolOnlyMode - ? await this.prisma.$queryRawUnsafe<{ id: string }[]>( - `SELECT m.id FROM memories m + ? await this.prisma.$queryRawUnsafe( + `SELECT m.id, + ts_rank(to_tsvector('english', m.raw), websearch_to_tsquery('english', $2)) AS ts_rank + FROM memories m WHERE m.id IN ( SELECT mpm.memory_id FROM memory_pool_memberships mpm WHERE mpm.pool_id = ANY($1::text[]) @@ -405,8 +422,10 @@ export class MemoryQueryService { searchQuery, ) : resolvedUserIds.length > 0 - ? await this.prisma.$queryRawUnsafe<{ id: string }[]>( - `SELECT id FROM memories + ? await this.prisma.$queryRawUnsafe( + `SELECT id, + ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $2)) AS ts_rank + FROM memories WHERE user_id = ANY($1::text[]) AND to_tsvector('english', raw) @@ websearch_to_tsquery('english', $2) AND deleted_at IS NULL @@ -419,8 +438,10 @@ export class MemoryQueryService { resolvedUserIds, searchQuery, ) - : await this.prisma.$queryRawUnsafe<{ id: string }[]>( - `SELECT id FROM memories + : await this.prisma.$queryRawUnsafe( + `SELECT id, + ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $1)) AS ts_rank + FROM memories WHERE to_tsvector('english', raw) @@ websearch_to_tsquery('english', $1) AND deleted_at IS NULL AND superseded_by_id IS NULL @@ -431,23 +452,34 @@ export class MemoryQueryService { LIMIT 100`, searchQuery, ); - // RRF fusion (k=60): BM25 rank contributes 1/(k+rank) so rank-1 - // BM25 hit scores ≈0.016, rank-100 ≈0.006. This prevents a flat - // 0.75 override from promoting low-quality exact-keyword matches - // above high-quality semantic matches. - const RRF_K = 60; + // Continuous FTS rescue band. Previously every FTS-rescued candidate + // was assigned the constant 1.25, which is what made 18/20 benchmark + // queries end in a 5-way tie at the top score. The score now varies + // with the ts_rank the SQL was already ordering by (SELECTed above but + // formerly discarded), blended with an RRF positional term. + // + // Invariant: band top is exactly 1.25 for the best hit, values are + // strictly decreasing with rank, and never fall to or below 1.10 — + // so FTS rescue still outranks ILIKE rescue and every cosine hit. + // Rows arrive ordered by ts_rank DESC, so row 0 carries the max. + const maxTsRank = + ftsResults.length > 0 ? Number(ftsResults[0].ts_rank) : 0; let ftsAdded = 0; for (let ftsRank = 0; ftsRank < ftsResults.length; ftsRank++) { const row = ftsResults[ftsRank]; - const bm25Score = 1 / (RRF_K + ftsRank + 1); + const bandedScore = ftsRescueScore( + Number(row.ts_rank), + maxTsRank, + ftsRank, + ); ftsResultIds.add(row.id); keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { - scoreMap.set(row.id, 1.25); + scoreMap.set(row.id, bandedScore); memoryIds.push(row.id); ftsAdded++; } else { - scoreMap.set(row.id, Math.max(scoreMap.get(row.id)!, 1.25)); + scoreMap.set(row.id, Math.max(scoreMap.get(row.id)!, bandedScore)); } } if (ftsAdded > 0) { @@ -468,10 +500,22 @@ export class MemoryQueryService { const ilikeConditions = words .map((_, i) => `LOWER(raw) LIKE $${i + paramOffset}`) .join(' OR '); + // Lexical coverage: how many of the extracted rescue terms this row + // actually contains. This is the quality signal that makes the + // ILIKE band continuous instead of a flat 1.1 constant. + const matchCountExpr = (column: string, offset: number): string => + words + .map( + (_, i) => + `(CASE WHEN LOWER(${column}) LIKE $${i + offset} THEN 1 ELSE 0 END)`, + ) + .join(' + '); const ilikeParams = words.map((w) => `%${w}%`); + type IlikeRow = { id: string; match_count: number }; const ilikeResults = poolOnlyMode - ? await this.prisma.$queryRawUnsafe<{ id: string }[]>( - `SELECT m.id FROM memories m + ? await this.prisma.$queryRawUnsafe( + `SELECT m.id, (${matchCountExpr('m.raw', 2)}) AS match_count + FROM memories m WHERE m.id IN ( SELECT mpm.memory_id FROM memory_pool_memberships mpm WHERE mpm.pool_id = ANY($1::text[]) @@ -490,8 +534,9 @@ export class MemoryQueryService { ...ilikeParams, ) : hasResolvedUsers - ? await this.prisma.$queryRawUnsafe<{ id: string }[]>( - `SELECT id FROM memories + ? await this.prisma.$queryRawUnsafe( + `SELECT id, (${matchCountExpr('raw', paramOffset)}) AS match_count + FROM memories WHERE user_id = ANY($1::text[]) AND (${ilikeConditions}) AND deleted_at IS NULL @@ -504,8 +549,9 @@ export class MemoryQueryService { resolvedUserIds, ...ilikeParams, ) - : await this.prisma.$queryRawUnsafe<{ id: string }[]>( - `SELECT id FROM memories + : await this.prisma.$queryRawUnsafe( + `SELECT id, (${matchCountExpr('raw', paramOffset)}) AS match_count + FROM memories WHERE (${ilikeConditions}) AND deleted_at IS NULL AND superseded_by_id IS NULL @@ -516,16 +562,37 @@ export class MemoryQueryService { LIMIT 20`, ...ilikeParams, ); + // Continuous ILIKE rescue band, replacing the flat 1.1 constant. + // Quality signal = lexical coverage (matched terms / extracted + // terms); positional tie-break = RRF over the SQL ordering + // (importance_score DESC, created_at DESC). + // + // Invariant: values live strictly inside (1.00, 1.10] — always + // above the cosine ceiling of 1.0, never reaching the FTS band — + // and are strictly decreasing with rank at equal coverage. let ilikeAdded = 0; - for (const row of ilikeResults) { + for ( + let ilikeRank = 0; + ilikeRank < ilikeResults.length; + ilikeRank++ + ) { + const row = ilikeResults[ilikeRank]; + const bandedScore = ilikeRescueScore( + Number(row.match_count), + words.length, + ilikeRank, + ); ftsResultIds.add(row.id); keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { - scoreMap.set(row.id, 1.1); + scoreMap.set(row.id, bandedScore); memoryIds.push(row.id); ilikeAdded++; } else { - scoreMap.set(row.id, Math.max(scoreMap.get(row.id)!, 1.1)); + scoreMap.set( + row.id, + Math.max(scoreMap.get(row.id)!, bandedScore), + ); } } if (ilikeAdded > 0) { @@ -620,10 +687,11 @@ export class MemoryQueryService { return { ...memory, score: semanticScore, + vectorScore: cosineMap.get(memory.id), __keywordRescued: keywordRescueIds.has(memory.id), } as MemoryWithScore; }) - .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)); + .sort(compareByRankKeys); const RERANK_POOL = sorted.length; @@ -720,15 +788,22 @@ export class MemoryQueryService { // Exact keyword/ILIKE rescued memories are deterministic high-signal hits. // Keep them sticky after reranking so the cross-encoder cannot drop fresh // exact-match writes from the final top-N. + // Preserve the banded rescue score the candidate already carries. Clamping + // to a flat 1.1 here would re-flatten exactly the scores the FTS/ILIKE + // bands above made continuous; the band bottom is only a floor for the + // pathological case of a rescued memory arriving with no score at all. const missingKeywordHits = [...keywordRescueMap.entries()] .filter(([id]) => !scoredMemories.some((m) => m.id === id)) .map( ([, mem]) => - ({ ...mem, score: Math.max(mem.score ?? 0, 1.1) }) as MemoryWithScore, + ({ + ...mem, + score: mem.score ?? ILIKE_RESCUE_BAND_BOTTOM, + }) as MemoryWithScore, ); if (missingKeywordHits.length > 0) { scoredMemories = [...missingKeywordHits, ...scoredMemories] - .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) + .sort(compareByRankKeys) .slice(0, limit); } @@ -747,7 +822,7 @@ export class MemoryQueryService { } return m; }); - scoredMemories.sort((a, b) => (b.score ?? 0) - (a.score ?? 0)); + scoredMemories.sort(compareByRankKeys); } let result: MemoryWithScore[] = this.filterRecallSurvivors(scoredMemories); @@ -911,9 +986,13 @@ export class MemoryQueryService { const adjustedScore = blendedScore * this.recallWeightService.recallWeight(memory); - return { ...memory, score: adjustedScore } as MemoryWithScore; + return { + ...memory, + score: adjustedScore, + vectorScore: scoreMap.get(memory.id), + } as MemoryWithScore; }) - .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)); + .sort(compareByRankKeys); // Apply the same post-processing chain as the standard path so multi-query // is not silently downgraded to cosine-only ranking (retrieval H4), then diff --git a/src/memory/memory-ranking.util.spec.ts b/src/memory/memory-ranking.util.spec.ts new file mode 100644 index 00000000..6a7c4ff6 --- /dev/null +++ b/src/memory/memory-ranking.util.spec.ts @@ -0,0 +1,217 @@ +import { + compareByRankKeys, + sortByRank, + ftsRescueScore, + ilikeRescueScore, + rrfNorm, + FTS_RESCUE_BAND_TOP, + FTS_RESCUE_BAND_BOTTOM, + ILIKE_RESCUE_BAND_TOP, + ILIKE_RESCUE_BAND_BOTTOM, + RankableMemory, +} from './memory-ranking.util'; + +describe('memory-ranking.util', () => { + describe('rrfNorm', () => { + it('is 1 at rank 0 and strictly decreasing', () => { + expect(rrfNorm(0)).toBe(1); + for (let r = 1; r < 100; r++) { + expect(rrfNorm(r)).toBeLessThan(rrfNorm(r - 1)); + expect(rrfNorm(r)).toBeGreaterThan(0); + } + }); + }); + + describe('ftsRescueScore', () => { + it('puts the best FTS hit exactly at the band top (1.25)', () => { + expect(ftsRescueScore(0.42, 0.42, 0)).toBeCloseTo( + FTS_RESCUE_BAND_TOP, + 10, + ); + expect(ftsRescueScore(0.42, 0.42, 0)).toBe(1.25); + }); + + it('gives DIFFERENT scores to candidates with different ts_rank', () => { + // This is the core defect: both of these used to be exactly 1.25. + const strong = ftsRescueScore(0.9, 0.9, 0); + const weak = ftsRescueScore(0.1, 0.9, 1); + expect(strong).not.toBe(weak); + expect(strong).toBeGreaterThan(weak); + }); + + it('is strictly decreasing with rank even when ts_rank ties exactly', () => { + // Near-duplicate distractors routinely produce identical ts_rank; the + // RRF positional term is what stops them collapsing into a tie. + const scores = [0, 1, 2, 3, 4].map((r) => ftsRescueScore(0.5, 0.5, r)); + for (let i = 1; i < scores.length; i++) { + expect(scores[i]).toBeLessThan(scores[i - 1]); + } + expect(new Set(scores).size).toBe(scores.length); + }); + + it('stays inside the (1.10, 1.25] band for any input', () => { + for (let r = 0; r < 100; r++) { + const s = ftsRescueScore(Math.random() * 0.9, 0.9, r); + expect(s).toBeGreaterThan(FTS_RESCUE_BAND_BOTTOM); + expect(s).toBeLessThanOrEqual(FTS_RESCUE_BAND_TOP); + } + // Degenerate inputs must not escape the band either. + expect(ftsRescueScore(NaN, 0, 0)).toBeGreaterThan(FTS_RESCUE_BAND_BOTTOM); + expect(ftsRescueScore(NaN, 0, 0)).toBeLessThanOrEqual( + FTS_RESCUE_BAND_TOP, + ); + expect(ftsRescueScore(0, 1, 99)).toBeGreaterThan(FTS_RESCUE_BAND_BOTTOM); + }); + + it('still outranks a near-perfect 0.99 cosine hit', () => { + expect(ftsRescueScore(0.001, 0.9, 99)).toBeGreaterThan(0.99); + expect(ftsRescueScore(0.9, 0.9, 0)).toBeGreaterThan(0.99); + }); + }); + + describe('ilikeRescueScore', () => { + it('scores full coverage at rank 0 exactly at the band top (1.10)', () => { + expect(ilikeRescueScore(3, 3, 0)).toBeCloseTo(ILIKE_RESCUE_BAND_TOP, 10); + }); + + it('ranks higher lexical coverage above lower coverage', () => { + expect(ilikeRescueScore(3, 3, 5)).toBeGreaterThan( + ilikeRescueScore(1, 3, 0), + ); + }); + + it('is strictly decreasing with rank at equal coverage', () => { + const scores = [0, 1, 2, 3].map((r) => ilikeRescueScore(2, 3, r)); + for (let i = 1; i < scores.length; i++) { + expect(scores[i]).toBeLessThan(scores[i - 1]); + } + }); + + it('stays strictly inside (1.00, 1.10] and below the FTS band', () => { + for (let r = 0; r < 40; r++) { + const s = ilikeRescueScore((r % 3) + 1, 3, r); + expect(s).toBeGreaterThan(ILIKE_RESCUE_BAND_BOTTOM); + expect(s).toBeLessThanOrEqual(ILIKE_RESCUE_BAND_TOP); + // Inter-band policy: FTS rescue > ILIKE rescue > cosine. + expect(s).toBeLessThan(ftsRescueScore(0.0001, 1, 999)); + expect(s).toBeGreaterThan(1.0); + } + }); + + it('handles a zero/garbled match count without collapsing to the floor', () => { + expect(ilikeRescueScore(0, 3, 0)).toBeGreaterThan( + ILIKE_RESCUE_BAND_BOTTOM, + ); + expect(ilikeRescueScore(0, 0, 0)).toBeGreaterThan( + ILIKE_RESCUE_BAND_BOTTOM, + ); + }); + }); + + describe('compareByRankKeys', () => { + const day = (n: number) => new Date(2026, 0, n); + + it('orders by score descending first', () => { + const out = sortByRank([ + { id: 'a', score: 0.5 }, + { id: 'b', score: 0.9 }, + ]); + expect(out.map((m) => m.id)).toEqual(['b', 'a']); + }); + + it('breaks score ties on cosine similarity', () => { + const out = sortByRank([ + { id: 'a', score: 1.25, vectorScore: 0.2 }, + { id: 'b', score: 1.25, vectorScore: 0.8 }, + ]); + expect(out.map((m) => m.id)).toEqual(['b', 'a']); + }); + + it('sorts candidates with no vector hit after ones that have one', () => { + const out = sortByRank([ + { id: 'a', score: 1.25 }, + { id: 'b', score: 1.25, vectorScore: 0 }, + ]); + expect(out.map((m) => m.id)).toEqual(['b', 'a']); + }); + + it('falls through to importance, then createdAt, then id', () => { + expect( + sortByRank([ + { id: 'a', score: 1, vectorScore: 0.5, importanceScore: 0.2 }, + { id: 'b', score: 1, vectorScore: 0.5, importanceScore: 0.9 }, + ]).map((m) => m.id), + ).toEqual(['b', 'a']); + + expect( + sortByRank([ + { + id: 'a', + score: 1, + vectorScore: 0.5, + importanceScore: 0.5, + createdAt: day(1), + }, + { + id: 'b', + score: 1, + vectorScore: 0.5, + importanceScore: 0.5, + createdAt: day(9), + }, + ]).map((m) => m.id), + ).toEqual(['b', 'a']); + + expect( + sortByRank([ + { + id: 'zzz', + score: 1, + vectorScore: 0.5, + importanceScore: 0.5, + createdAt: day(1), + }, + { + id: 'aaa', + score: 1, + vectorScore: 0.5, + importanceScore: 0.5, + createdAt: day(1), + }, + ]).map((m) => m.id), + ).toEqual(['aaa', 'zzz']); + }); + + it('is order-independent: identical sets in different input orders sort identically', () => { + // The defect: with a bare score comparator + stable sort, these two + // permutations returned different top-1 results purely because of the + // order Postgres handed the rows back in. + const tied: RankableMemory[] = [ + { id: 'm3', score: 1.25, importanceScore: 0.5, createdAt: day(2) }, + { id: 'm1', score: 1.25, importanceScore: 0.5, createdAt: day(2) }, + { id: 'm5', score: 1.25, importanceScore: 0.5, createdAt: day(2) }, + { id: 'm2', score: 1.25, importanceScore: 0.5, createdAt: day(2) }, + { id: 'm4', score: 1.25, importanceScore: 0.5, createdAt: day(2) }, + ]; + const reversed = [...tied].reverse(); + const shuffled = [tied[2], tied[0], tied[4], tied[1], tied[3]]; + + const a = sortByRank(tied).map((m) => m.id); + const b = sortByRank(reversed).map((m) => m.id); + const c = sortByRank(shuffled).map((m) => m.id); + + expect(a).toEqual(b); + expect(a).toEqual(c); + expect(a).toEqual(['m1', 'm2', 'm3', 'm4', 'm5']); + }); + + it('is a consistent comparator (antisymmetric)', () => { + const x = { id: 'a', score: 1, vectorScore: 0.5 }; + const y = { id: 'b', score: 1, vectorScore: 0.5 }; + expect(Math.sign(compareByRankKeys(x, y))).toBe( + -Math.sign(compareByRankKeys(y, x)), + ); + expect(compareByRankKeys(x, x)).toBe(0); + }); + }); +}); diff --git a/src/memory/memory-ranking.util.ts b/src/memory/memory-ranking.util.ts new file mode 100644 index 00000000..6b93bbe3 --- /dev/null +++ b/src/memory/memory-ranking.util.ts @@ -0,0 +1,182 @@ +/** + * Shared recall ranking primitives. + * + * Two defects motivated this module (see + * docs/research/memory-formation-query-transform/04-finding-tie-domination.md): + * + * 1. The keyword-rescue paths in `memory-query.service.ts` assigned *constants* + * (1.25 for FTS/BM25, 1.1 for ILIKE). Every rescued candidate therefore had + * an identical score, so on a noisy corpus 18/20 queries ended in a + * 5-or-more-way tie at the top score. + * 2. Every score sort was a bare `(b.score ?? 0) - (a.score ?? 0)`. Because + * `Array.prototype.sort` is stable, ties were resolved by whatever order + * Postgres happened to return rows in — i.e. top-1 was decided by row order, + * not by relevance. + * + * This module fixes both: continuous *banded* rescue scores, and one explicit + * deterministic comparator used by every score sort in the recall pipeline. + */ + +/** + * Score bands. Rescue hits deliberately sit ABOVE the cosine ceiling of 1.0 — + * that inter-band priority is existing, intentional policy ("exact match + * agrees" beats "semantically close") and is preserved verbatim here. Only the + * ordering *within* each band changes: it used to be a constant, it is now + * continuous. + * + * cosine (0.00 .. 1.00] + * ILIKE rescue (1.00 .. 1.10] + * FTS rescue (1.10 .. 1.25] + * + * The bands are half-open at the bottom and closed at the top, so they never + * overlap and a rescue hit can never be demoted below the band it belongs to. + */ +export const ILIKE_RESCUE_BAND_BOTTOM = 1.0; +export const ILIKE_RESCUE_BAND_TOP = 1.1; +export const FTS_RESCUE_BAND_BOTTOM = 1.1; +export const FTS_RESCUE_BAND_TOP = 1.25; + +/** RRF constant retained from the original (dead) BM25 fusion code. */ +export const RRF_K = 60; + +/** + * Positional term derived from Reciprocal Rank Fusion, normalised so that + * rank 0 → 1 and it decreases *strictly* monotonically with rank: + * + * rrf(r) = 1 / (RRF_K + r + 1) + * rrfNorm(r) = rrf(r) / rrf(0) = (RRF_K + 1) / (RRF_K + 1 + r) ∈ (0, 1] + * + * This is what guarantees strictness: the quality signals below (ts_rank, + * term coverage) can legitimately tie between near-duplicate memories, which + * is exactly the regime the tie-domination finding is about. + */ +export function rrfNorm(rankIndex: number): number { + return (RRF_K + 1) / (RRF_K + 1 + Math.max(0, rankIndex)); +} + +/** Weight split between the quality signal and the positional (RRF) signal. */ +const QUALITY_WEIGHT = 0.9; +const POSITION_WEIGHT = 0.1; + +/** + * Map a Postgres `ts_rank` value into the FTS rescue band. + * + * Invariants: + * - the best FTS hit (rank 0, ts_rank == max) scores exactly + * `FTS_RESCUE_BAND_TOP` (1.25), so the pre-existing inter-band policy is + * unchanged; + * - the value is strictly decreasing in `rankIndex` (rows arrive ordered by + * `ts_rank DESC`), so two rescued candidates never tie; + * - the value is always > `FTS_RESCUE_BAND_BOTTOM` (1.10), i.e. an FTS rescue + * always outranks every ILIKE rescue and every cosine hit. + * + * `quality` is ts_rank normalised by the max ts_rank in the result set, so it + * reflects *how well* the row matched rather than merely that it matched. + */ +export function ftsRescueScore( + tsRank: number, + maxTsRank: number, + rankIndex: number, +): number { + const safeTsRank = Number.isFinite(tsRank) ? Math.max(0, tsRank) : 0; + const safeMax = Number.isFinite(maxTsRank) ? maxTsRank : 0; + const quality = safeMax > 0 ? Math.min(1, safeTsRank / safeMax) : 1; + const blended = + QUALITY_WEIGHT * quality + POSITION_WEIGHT * rrfNorm(rankIndex); + return ( + FTS_RESCUE_BAND_BOTTOM + + (FTS_RESCUE_BAND_TOP - FTS_RESCUE_BAND_BOTTOM) * blended + ); +} + +/** + * Map an ILIKE lexical-rescue hit into the ILIKE band. + * + * The ILIKE SQL orders by `importance_score DESC, created_at DESC`, so unlike + * FTS there is no relevance value to normalise. The quality signal is instead + * *lexical coverage*: how many of the extracted rescue terms this row actually + * contains. Coverage is computed in SQL (`SUM(CASE WHEN LOWER(raw) LIKE $n …)`) + * so a row matching 3 of 3 terms outranks one matching 1 of 3. + * + * Invariants: + * - the value is strictly inside `(1.00, 1.10]`, so an ILIKE rescue always + * outranks any cosine hit (ceiling 1.0) and never reaches the FTS band; + * - full coverage at rank 0 scores exactly `ILIKE_RESCUE_BAND_TOP` (1.10); + * - the value is strictly decreasing in `rankIndex` at equal coverage. + */ +export function ilikeRescueScore( + matchedTerms: number, + totalTerms: number, + rankIndex: number, +): number { + const total = totalTerms > 0 ? totalTerms : 1; + // A returned row matched at least one term by construction; clamp defensively + // so a missing/garbled match count can never collapse the row to the floor. + const matched = Math.min(total, Math.max(1, Math.floor(matchedTerms) || 1)); + const coverage = matched / total; + const blended = + QUALITY_WEIGHT * coverage + POSITION_WEIGHT * rrfNorm(rankIndex); + return ( + ILIKE_RESCUE_BAND_BOTTOM + + (ILIKE_RESCUE_BAND_TOP - ILIKE_RESCUE_BAND_BOTTOM) * blended + ); +} + +/** Minimal shape the comparator needs. Deliberately structural, not `Memory`. */ +export interface RankableMemory { + id?: string; + score?: number | null; + /** Raw cosine similarity from the vector search, when this candidate had one. */ + vectorScore?: number | null; + importanceScore?: number | null; + createdAt?: Date | string | null; +} + +function timeValue(value: Date | string | null | undefined): number { + if (value == null) return 0; + const t = value instanceof Date ? value.getTime() : new Date(value).getTime(); + return Number.isFinite(t) ? t : 0; +} + +/** + * The single deterministic ordering used by every score sort in the recall + * pipeline. Key order: + * + * 1. score desc + * 2. vectorScore desc (cosine similarity, when available; candidates + * with no vector hit sort after ones that have + * one, since cosine ∈ [0,1] and missing → -1) + * 3. importanceScore desc + * 4. createdAt desc (newer first) + * 5. id asc (total order — this is the key that removes the + * dependency on Postgres row order entirely) + * + * Because key 5 is unique, the comparator induces a *total* order: the same set + * of candidates sorts identically regardless of the input array order. That is + * the actual defect being fixed. + */ +export function compareByRankKeys( + a: RankableMemory, + b: RankableMemory, +): number { + const scoreDelta = (b.score ?? 0) - (a.score ?? 0); + if (scoreDelta !== 0) return scoreDelta; + + const vectorDelta = (b.vectorScore ?? -1) - (a.vectorScore ?? -1); + if (vectorDelta !== 0) return vectorDelta; + + const importanceDelta = (b.importanceScore ?? 0) - (a.importanceScore ?? 0); + if (importanceDelta !== 0) return importanceDelta; + + const createdDelta = timeValue(b.createdAt) - timeValue(a.createdAt); + if (createdDelta !== 0) return createdDelta; + + const idA = a.id ?? ''; + const idB = b.id ?? ''; + return idA < idB ? -1 : idA > idB ? 1 : 0; +} + +/** Non-mutating convenience wrapper around {@link compareByRankKeys}. */ +export function sortByRank(items: T[]): T[] { + return [...items].sort(compareByRankKeys); +} diff --git a/src/memory/memory.types.ts b/src/memory/memory.types.ts index fa056d95..610a5c9a 100644 --- a/src/memory/memory.types.ts +++ b/src/memory/memory.types.ts @@ -63,6 +63,12 @@ export interface MemoryWithExtraction extends Memory { export interface MemoryWithScore extends MemoryWithExtraction { score?: number; + /** + * Raw cosine similarity from the vector search, preserved even after `score` + * has been overridden by a keyword-rescue band or a reranker blend. Used as + * the first tie-break key by `compareByRankKeys` (memory-ranking.util.ts). + */ + vectorScore?: number; /** Present when this memory was surfaced by the Anticipatory Recall Engine. */ recallSource?: 'standard' | 'anticipatory' | 'graph'; /** Anticipatory metadata (strategy, reason, salience). Only present when recallSource='anticipatory'. */ From 6f4699ba0d049b5f63a74a660cdfbca7ad6e1ee1 Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 17:54:32 -0700 Subject: [PATCH 5/8] research(recall): relative lexical rescue behind RECALL_RELATIVE_RESCUE (default off) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rescue score bands added in c905438 do not merely break ties in favour of lexical hits — they replace the ranking. Measured on the 20-task noisy prefix corpus with a new retrieval-only probe (no LLM, no generation): - gold is in the candidate pool for 20/20 tasks, so this is a ranking failure, not a recall failure; - inside an FTS-rescued block the candidate with the *highest* cosine is ranked *last*, because ftsRescueScore() overwrites scoreMap with a ts_rank-derived value and the cosine only survives as an unused tie-break key; - the sticky keyword re-add re-injects rescued memories with their raw pre-rerank band scores (1.25/1.15/1.05) after applyReranking has rescaled every other candidate to <= ~1.0, so band values are sorted against rescaled values and win categorically. This also makes results non-monotonic in `limit`: the same query returns 5 band hits at limit=5 and zero band values at limit=50. Prototype fix, off by default (RECALL_RELATIVE_RESCUE=true to enable): relativeRescueScore() turns lexical agreement into a bounded boost on the candidate's own cosine (score = cosine * (1 + maxBoost * quality); 0.2 FTS, 0.1 ILIKE, 0.15 identity) instead of an absolute band above the cosine ceiling. Lexical-only candidates with no vector hit are anchored at 0.9 * bestCosine so the fresh-exact-write guarantee survives without letting them displace the best semantic hit. The sticky re-add appends below the rescaled floor in relative mode instead of above it. Also under the flag: the flat 1.15 identity rescue constant (same flat-constant defect class c905438 fixed) becomes continuous. The flat 0.75 forcedFts constant is annotated, not changed — topIds and memoryMap are both built from `sorted`, so that branch is unreachable and forcedFts is always empty. Measured, both arms against usage counters reset to the same cold snapshot (recall increments retrievalCount and applyUsageWeighting feeds it back into the score, so the benchmark perturbs itself): gold in top-5 6/20 -> 7/20 gold in top-10 6/20 -> 8/20 MRR@10 0.150 -> 0.174 gold in top-1 0/20 -> 0/20 Small, and deliberately reported as such: after the fix the ranking is the semantic ranking, and on 11 of 20 tasks the gold memory sits behind all ten of its distractors in that ordering. The distractors restate the query verbatim; under bge-base they are the closer match. The band fix is worth having for determinism, scale hygiene and limit-monotonicity, not for benchmark score. Default behaviour unchanged: src/memory 66 suites / 1150 tests pass. Analysis in docs/research/memory-formation-query-transform/05-finding-band-inversion.md. Co-Authored-By: Claude Opus 4.7 --- .../05-finding-band-inversion.md | 195 ++++++++++++++++++ scripts/research/retrieval-probe.mjs | 173 ++++++++++++++++ scripts/research/run-rescue-ab.sh | 54 +++++ src/memory/memory-query.service.ts | 140 +++++++++++-- src/memory/memory-ranking.util.spec.ts | 98 +++++++++ src/memory/memory-ranking.util.ts | 80 +++++++ 6 files changed, 722 insertions(+), 18 deletions(-) create mode 100644 docs/research/memory-formation-query-transform/05-finding-band-inversion.md create mode 100644 scripts/research/retrieval-probe.mjs create mode 100755 scripts/research/run-rescue-ab.sh diff --git a/docs/research/memory-formation-query-transform/05-finding-band-inversion.md b/docs/research/memory-formation-query-transform/05-finding-band-inversion.md new file mode 100644 index 00000000..75ecca02 --- /dev/null +++ b/docs/research/memory-formation-query-transform/05-finding-band-inversion.md @@ -0,0 +1,195 @@ +# Finding: The Rescue Band Inverts Ranking, But It Is Not Why Gold Is Missing + +**Date:** 2026-08-23 +**Status:** measured on the noisy prefix corpus (retrieval-only, no LLM) +**Follows:** [`04-finding-tie-domination.md`](04-finding-tie-domination.md), fixed in `c905438` +**Instance:** research worktree @ `127.0.0.1:3007`, DB `engram_ablation_prefix`, +`RERANK_ENABLED` unset (so `applyReranking` takes the fallback-blend path), +`USAGE_WEIGHT=0.15`, corpus = 20 gold + 200 distractors, `userId +mnemon-v02-noise-prefix` + +--- + +## TL;DR + +1. **Gold is never missing from the candidate pool.** 20/20 tasks retrieve the + gold memory. This is purely a *ranking* failure, not a recall failure. The + two failure modes need different fixes and we have the ranking one. +2. **The band is a real defect** — it discards the cosine ordering, mixes score + scales, and makes results non-monotonic in `limit`. Fixed under a flag. +3. **But the band is not the reason gold misses top-5.** With the band replaced + by a relative boost, gold-in-top-5 moves 6/20 → 7/20. For 11 of the 20 tasks + the gold memory is *behind all ten of its distractors in the semantic + ordering too*. The premise "the gold memory can be the best semantic match + and still never appear in the top-5" does not hold on this corpus; on the + majority of tasks the gold is not the best semantic match. + +## 1. Where the gold actually ranks + +`scripts/research/retrieval-probe.mjs` replays the 20 task queries against a +live instance and reports, per query, the top-10 with the band each hit came +from and where the gold landed — both in the page the benchmark consumes +(`limit=10`) and in the full 221-memory pool. + +| Measure (20 tasks, `limit=10`) | band (shipped) | relative (flag on) | +|---|---|---| +| gold anywhere in the candidate pool | **20/20** | **20/20** | +| gold in top-1 | 0/20 | 0/20 | +| gold in top-5 | 6/20 | 7/20 | +| gold in top-10 | 6/20 | 8/20 | +| MRR@10 | 0.150 | 0.174 | + +Both arms were run against usage counters reset to the same cold snapshot +(`zz_probe_usage_snapshot`), because recall increments `retrieval_count` and +`applyUsageWeighting` feeds it straight back into the score — see §5. + +## 2. The band inverts the cosine ordering it sits on top of + +The decisive observation. For query *"…app config environment variables and +parsing"*, at `limit=250` (shipped code), the returned order was: + +``` +rank score vectorScore +1 0.42768 0.78597 +2 0.42761 0.78600 +... +10 0.42711 0.78982 <- highest cosine in the set, ranked last +``` + +Every one of those is an FTS-rescued distractor. Their final scores descend +monotonically while their cosines do not: the candidate with the **best** cosine +in the block (0.78982) is ranked **last** in the block. `ftsRescueScore()` +derives the band value from `ts_rank` and the RRF position and *overwrites* +`scoreMap`, so the semantic signal survives only as tie-break key #2 in +`compareByRankKeys` — and it never gets used, because the band values are +already distinct. Lexical rank does not merely win the tie; it replaces the +ordering. + +Code path: `src/memory/memory-query.service.ts:478-500` (FTS, +`scoreMap.set(row.id, bandedScore)` on both the insert and the `Math.max` +branch), `:596-616` (ILIKE), `src/memory/memory-ranking.util.ts:76-123`. + +## 3. What actually decides the top-5: post-rerank scale mixing + +`applyReranking` (`src/memory/memory-query-ranking.service.ts:226-307`) rescales +**every** candidate. With the cross-encoder it emits `normalisedRerank * 0.85 + +importance * 0.15`; without it, the fallback blend is `(score * 0.85 + +importance * 0.15) * importanceMultiplier * sentimentPenalty`. Either way the +output is ≤ ~1.0 and usually ≤ 0.5. + +The sticky keyword re-add immediately afterwards +(`src/memory/memory-query.service.ts:862-905`) re-injects every rescued memory +that fell out of the reranked top-N **with its raw pre-rerank band score** +(1.25 / 1.15 / 1.05), taken from `keywordRescueMap`. Those raw values are then +sorted against rescaled values. They win categorically. The final top-N is +whatever the lexical rescue produced, and the reranker's judgement is discarded. + +This is directly observable as a **non-monotonicity in `limit`** — same query, +same instance, seconds apart: + +| `limit` | top-1 score | what the page contains | +|---|---|---| +| 5 | 1.25000 | 5 FTS-band distractors | +| 10 | 1.24908 | 1 FTS-band + 9 ILIKE-band distractors | +| 50 | 0.90922 | pure reranked/blended scores, no band values at all | +| 250 | 0.90051 | pure reranked/blended scores | + +At `limit=50` nothing is "missing" from the reranked top-50, so +`missingKeywordHits` is empty and the re-add never fires. **Asking for more +results changes which results rank first.** That is a correctness bug on its +own, independent of the band policy. + +## 4. The two remaining flat constants + +- **Identity rescue, `memory-query.service.ts:684/688` (pre-fix line 642/646)** — + flat `1.15` for all ten rows. Confirmed live: `1.15` sits *inside* the FTS + band `(1.10, 1.25]`, so identity hits interleave with FTS hits, and all ten + identity rows tie with each other and fall back to `compareByRankKeys` keys + 2-5. Same defect class as the one `c905438` fixed. Fixed under the flag. +- **`forcedFts`, `memory-query.service.ts:774` (pre-fix line 709)** — flat + `0.75`, and **unreachable**. `topIds` and `memoryMap` are both built from + `sorted`, so `!topIds.has(id)` implies `memoryMap.get(id) === undefined` and + the inner `if (mem)` never fires; `forcedFts` is always `[]`. This is dead + code, not a live scoring path. Annotated in place rather than "fixed": + making it live would inject FTS-only candidates at a flat constant, i.e. + reintroduce exactly the defect `c905438` removed. + +## 5. Measurement hazard found along the way + +Every recall does `retrievalCount: { increment: 1 }` for every returned memory +(`memory-query.service.ts:912-920`), and `applyUsageWeighting` blends +`usageSignal` (a function of `retrievalCount`) back into the score at weight +0.15 once `retrievalCount >= 3`. Retrieval therefore reinforces whatever it +already returns — and on this corpus what it already returns is distractors. It +also means **the benchmark perturbs itself**: the same query, run twice, does +not return the same ranking. Any A/B on this corpus must reset the counters +between arms, which `scripts/research/run-rescue-ab.sh` does. + +## 6. Should a lexical hit ever outrank a strong semantic hit? + +**No — not categorically, which is what the band does.** But the band is not +merely a tuning error; there is a real use case underneath it, and removing it +outright would regress that case. + +The case it serves: a *fresh exact-match write*. A memory ingested seconds ago +may have a stale, missing, or low-quality embedding, and a user querying an +exact rare token ("what's my Vercel project id", "ORD-4471") wants that memory +even though its cosine is unremarkable. The band guarantees it surfaces. That +guarantee is load-bearing — the sticky re-add comment says so explicitly. + +The flaw is that the guarantee is written as a **priority stratum** rather than +as **evidence**. Absolute bands above the cosine ceiling encode "any lexical +match beats every semantic match, at any strength, forever". That is only the +right answer when the lexical match is *rare* — a discriminating token. On a +corpus where the distractors restate the query, the lexical signal has zero +discriminating power and the band hands the whole top-N to noise. + +**Recommendation: make it relative, not absolute.** A lexical hit should buy a +bounded multiplicative promotion on the candidate's own semantic score: + +``` +score = cosine * (1 + maxBoost * lexicalQuality) +``` + +with `maxBoost` 0.2 for FTS, 0.1 for ILIKE, 0.15 for identity. Properties: + +- a lexical hit still **promotes** (that is the use case), but a candidate with + a materially better cosine can still beat it; +- the cosine ordering **inside** the rescued set is preserved instead of being + overwritten by `ts_rank`; +- everything stays in one numeric scale, so usage weighting, the fallback + blend and the cross-encoder compare like with like, and the sticky re-add + stops being a scale-mixing bug; +- lexical-only candidates (matched by FTS/ILIKE but absent from the vector + pool — the fresh-write case) get anchored at `0.9 * bestCosine`, so they are + admitted to the pool and are recoverable, but cannot displace the best + semantic hit. + +Rejected alternatives: *removing* the band loses the fresh-write guarantee; +*capping* it at a lower constant keeps the categorical inversion and just moves +the threshold; *hard-gating* rescue on IDF/rarity of the matched terms is +probably the right long-term answer but needs corpus statistics Engram does not +currently keep. + +## 7. What the fix does and does not buy + +Flag `RECALL_RELATIVE_RESCUE=true` (default `false`, no change to shipped +behaviour): + +- gold-in-top-5: 6/20 → 7/20 +- gold-in-top-10: 6/20 → 8/20 +- MRR@10: 0.150 → 0.174 +- top-1: 0/20 → 0/20 + +Small, because after the fix the ranking *is* the semantic ranking, and on 11 of +the 20 tasks the gold memory sits behind all ten of its distractors in that +ordering (`rankInPool == 11` in `/tmp/probe-true.json`). The distractors are +constructed as verbatim restatements of the query plus a disclaimer; the gold is +the query prefix plus the actual fact. Under `bge-base` the restatement is the +closer match. No reranking policy can recover that — it needs a different +embedding, an IDF-aware lexical gate, or a corpus that is not adversarial in +exactly this way. + +The honest read: **the band fix is worth shipping for determinism, scale +hygiene and `limit`-monotonicity, not for benchmark score.** The 5% gold-in-top-5 +number is dominated by embedding discrimination, not by the ranking policy. diff --git a/scripts/research/retrieval-probe.mjs b/scripts/research/retrieval-probe.mjs new file mode 100644 index 00000000..91605cd4 --- /dev/null +++ b/scripts/research/retrieval-probe.mjs @@ -0,0 +1,173 @@ +#!/usr/bin/env node +/** + * Retrieval-only probe for the noisy mnemon corpus. + * + * Replays the 20 task queries against a running Engram instance and reports, + * per query: the top-N with scores, the score band each hit fell into, and + * where the gold ("allowed") memory actually ranked. + * + * NO LLM. NO generation. NO writes to Engram. Pure recall measurement. + * + * Usage: + * node scripts/research/retrieval-probe.mjs [--limit 10] [--label before] + * + * Env: + * PROBE_BASE_URL default http://127.0.0.1:3007 + * PROBE_API_KEY required + * PROBE_USER_ID default mnemon-v02-noise-prefix + * PROBE_TASKS_DIR default /.mnemon/noisy-tasks-prefix + * PROBE_RECEIPT default /.mnemon/noisy-corpus-receipt-prefix.json + */ + +import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const MNEMON = process.env.PROBE_MNEMON ?? '/Users/beauxwalton/projects/mnemon'; +const BASE = (process.env.PROBE_BASE_URL ?? 'http://127.0.0.1:3007').replace(/\/+$/, ''); +const KEY = process.env.PROBE_API_KEY; +const USER = process.env.PROBE_USER_ID ?? 'mnemon-v02-noise-prefix'; +const TASKS = process.env.PROBE_TASKS_DIR ?? join(MNEMON, '.mnemon/noisy-tasks-prefix'); +const RECEIPT = process.env.PROBE_RECEIPT ?? join(MNEMON, '.mnemon/noisy-corpus-receipt-prefix.json'); + +if (!KEY) { + process.stderr.write('PROBE_API_KEY is required\n'); + process.exit(2); +} + +const args = process.argv.slice(2); +function arg(name, fallback) { + const i = args.indexOf(`--${name}`); + return i >= 0 && args[i + 1] !== undefined ? args[i + 1] : fallback; +} +const LIMIT = Number(arg('limit', '10')); +const LABEL = arg('label', 'probe'); +const OUT = arg('out', null); +const EXTRA_HEADERS = {}; +const flagHeader = arg('header', null); // e.g. --header 'X-Engram-Rescue-Mode: relative' +if (flagHeader) { + const idx = flagHeader.indexOf(':'); + EXTRA_HEADERS[flagHeader.slice(0, idx).trim()] = flagHeader.slice(idx + 1).trim(); +} + +const receipt = JSON.parse(readFileSync(RECEIPT, 'utf8')); +const goldByTask = receipt.targets; // { taskId: goldMemoryId } + +const taskFiles = readdirSync(TASKS) + .filter((f) => /^\d\d-.*\.json$/.test(f)) + .sort(); + +/** Which scoring band a returned score fell into. */ +function band(score) { + if (score === undefined || score === null) return 'none'; + if (score > 1.25 + 1e-9) return 'boosted>FTS'; // graph x1.2 etc + if (score > 1.1 + 1e-9) return 'FTS(1.10,1.25]'; + if (score > 1.0 + 1e-9) return 'ILIKE(1.00,1.10]'; + return 'vector<=1.00'; +} + +async function recall(query, limit) { + const res = await fetch(`${BASE}/v1/memories/query?scope=user`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-AM-API-Key': KEY, + 'X-AM-User-ID': USER, + ...EXTRA_HEADERS, + }, + body: JSON.stringify({ query, limit }), + }); + const text = await res.text(); + if (!res.ok) throw new Error(`recall ${res.status}: ${text.slice(0, 400)}`); + const data = JSON.parse(text); + return data.memories ?? []; +} + +const rows = []; +let goldTop1 = 0; +let goldTop5 = 0; +let goldTop10 = 0; +let goldInPool = 0; + +for (const file of taskFiles) { + const task = JSON.parse(readFileSync(join(TASKS, file), 'utf8')); + const gold = goldByTask[task.id]; + if (!gold) { + process.stderr.write(`no gold target for task ${task.id}\n`); + continue; + } + + // Ranked page the benchmark would actually consume. + const page = await recall(task.query, LIMIT); + // Deep page: is the gold anywhere in the candidate pool at all? + const deep = await recall(task.query, 250); + + const rank = page.findIndex((m) => m.id === gold); // -1 = absent + const deepRank = deep.findIndex((m) => m.id === gold); + const goldRow = deep[deepRank] ?? null; + + if (rank === 0) goldTop1++; + if (rank >= 0 && rank < 5) goldTop5++; + if (rank >= 0 && rank < 10) goldTop10++; + if (deepRank >= 0) goldInPool++; + + rows.push({ + task: task.id, + query: task.query, + goldId: gold, + rankInPage: rank, + rankInPool: deepRank, + poolSize: deep.length, + goldScore: goldRow ? goldRow.score : null, + goldBand: goldRow ? band(goldRow.score) : 'ABSENT', + top: page.slice(0, 10).map((m, i) => ({ + i, + id: m.id, + score: m.score, + band: band(m.score), + gold: m.id === gold, + text: (m.raw ?? m.content ?? '').slice(0, 70), + })), + }); +} + +const n = rows.length; +const summary = { + label: LABEL, + baseUrl: BASE, + limit: LIMIT, + headers: EXTRA_HEADERS, + tasks: n, + goldTop1: `${goldTop1}/${n}`, + goldTop5: `${goldTop5}/${n}`, + goldTop10: `${goldTop10}/${n}`, + goldAnywhereInPool: `${goldInPool}/${n}`, +}; + +// ── human-readable ──────────────────────────────────────────────────────── +process.stdout.write(`\n=== ${LABEL} (limit=${LIMIT}) ===\n`); +for (const r of rows) { + const where = + r.rankInPage >= 0 + ? `page rank ${r.rankInPage}` + : r.rankInPool >= 0 + ? `NOT in page; pool rank ${r.rankInPool}/${r.poolSize}` + : `NOT IN POOL AT ALL (pool ${r.poolSize})`; + process.stdout.write( + `\n${r.task}\n gold ${r.goldId} -> ${where} score=${ + r.goldScore === null ? 'n/a' : Number(r.goldScore).toFixed(5) + } band=${r.goldBand}\n`, + ); + for (const t of r.top) { + process.stdout.write( + ` ${String(t.i).padStart(2)} ${Number(t.score).toFixed(5)} ${t.band.padEnd(17)} ${ + t.gold ? 'GOLD ' : ' ' + }${t.text}\n`, + ); + } +} +process.stdout.write(`\n--- summary ---\n${JSON.stringify(summary, null, 2)}\n`); + +if (OUT) { + writeFileSync(OUT, JSON.stringify({ summary, rows }, null, 2)); + process.stdout.write(`wrote ${OUT}\n`); +} diff --git a/scripts/research/run-rescue-ab.sh b/scripts/research/run-rescue-ab.sh new file mode 100755 index 00000000..9dbc4f0b --- /dev/null +++ b/scripts/research/run-rescue-ab.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# A/B the RECALL_RELATIVE_RESCUE prototype flag on the noisy mnemon corpus. +# +# Retrieval-only. No LLM, no generation. Restarts the research Engram instance +# once per arm and resets the usage counters (retrieval_count / used_count / +# last_retrieved_at) between arms, because recall increments them and +# usage-weighted reranking then feeds them back into the score — without the +# reset the second arm is measured against a different system than the first. +# +# Usage: scripts/research/run-rescue-ab.sh +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KEY="${1:-${PROBE_API_KEY:-}}" +DB="${PROBE_DB_URL:-postgresql://engram:engram@localhost:5432/engram_ablation_prefix}" +PORT="${PROBE_PORT:-3007}" + +[ -n "$KEY" ] || { echo "usage: $0 " >&2; exit 2; } + +reset_usage() { + psql "$DB" -q -c " + UPDATE memories m + SET retrieval_count = s.retrieval_count, + used_count = s.used_count, + unused_count = s.unused_count, + last_retrieved_at = s.last_retrieved_at, + last_used_at = s.last_used_at + FROM zz_probe_usage_snapshot s + WHERE m.id = s.id;" >/dev/null +} + +start_server() { + local mode="$1" + pkill -f "node dist/main.js" 2>/dev/null || true + sleep 2 + cd "$REPO" + RECALL_RELATIVE_RESCUE="$mode" nohup node dist/main.js \ + >"/tmp/engram-research-server-${mode}.log" 2>&1 & + for _ in $(seq 1 60); do + if curl -sf -m 2 "http://127.0.0.1:${PORT}/v1/health" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + echo "server did not come up (mode=$mode)" >&2; exit 1 +} + +for MODE in false true; do + LABEL=$([ "$MODE" = "false" ] && echo BEFORE-band || echo AFTER-relative) + echo "### $LABEL (RECALL_RELATIVE_RESCUE=$MODE)" + start_server "$MODE" + reset_usage + PROBE_API_KEY="$KEY" node "$REPO/scripts/research/retrieval-probe.mjs" \ + --limit 10 --label "$LABEL" --out "/tmp/probe-${MODE}.json" \ + | tail -20 +done diff --git a/src/memory/memory-query.service.ts b/src/memory/memory-query.service.ts index 0df91017..42ef1f8d 100644 --- a/src/memory/memory-query.service.ts +++ b/src/memory/memory-query.service.ts @@ -47,11 +47,29 @@ import { ftsRescueScore, ilikeRescueScore, ILIKE_RESCUE_BAND_BOTTOM, + relativeRescueScore, + RELATIVE_FTS_MAX_BOOST, + RELATIVE_ILIKE_MAX_BOOST, + RELATIVE_IDENTITY_MAX_BOOST, } from './memory-ranking.util'; @Injectable() export class MemoryQueryService { private readonly logger = new Logger(MemoryQueryService.name); + + /** + * PROTOTYPE FLAG — default OFF, no change to shipped behaviour. + * + * `RECALL_RELATIVE_RESCUE=true` switches the keyword-rescue paths from the + * absolute score bands (FTS in (1.10,1.25], ILIKE in (1.00,1.10], identity a + * flat 1.15 — all of which sit above the cosine ceiling of 1.0 and therefore + * categorically outrank every semantic hit) to a bounded boost applied to the + * candidate's own cosine similarity. See + * docs/research/memory-formation-query-transform/05-finding-band-inversion.md. + */ + private readonly relativeRescue = + process.env.RECALL_RELATIVE_RESCUE === 'true'; + constructor( private prisma: PrismaService, private embedding: EmbeddingService, @@ -464,14 +482,26 @@ export class MemoryQueryService { // Rows arrive ordered by ts_rank DESC, so row 0 carries the max. const maxTsRank = ftsResults.length > 0 ? Number(ftsResults[0].ts_rank) : 0; + // Relative mode only: the best cosine in this query's vector pool, used + // to anchor lexical-only candidates below the best semantic hit. + const bestVector = vectorResults.reduce( + (best, r) => (r.score > best ? r.score : best), + 0, + ); let ftsAdded = 0; for (let ftsRank = 0; ftsRank < ftsResults.length; ftsRank++) { const row = ftsResults[ftsRank]; - const bandedScore = ftsRescueScore( - Number(row.ts_rank), - maxTsRank, - ftsRank, - ); + const quality = + maxTsRank > 0 ? Math.min(1, Number(row.ts_rank) / maxTsRank) : 1; + const bandedScore = this.relativeRescue + ? relativeRescueScore( + cosineMap.get(row.id), + quality, + ftsRank, + RELATIVE_FTS_MAX_BOOST, + bestVector, + ) + : ftsRescueScore(Number(row.ts_rank), maxTsRank, ftsRank); ftsResultIds.add(row.id); keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { @@ -577,11 +607,23 @@ export class MemoryQueryService { ilikeRank++ ) { const row = ilikeResults[ilikeRank]; - const bandedScore = ilikeRescueScore( - Number(row.match_count), - words.length, - ilikeRank, - ); + const coverage = + words.length > 0 + ? Math.min(1, Number(row.match_count) / words.length) + : 1; + const bandedScore = this.relativeRescue + ? relativeRescueScore( + cosineMap.get(row.id), + coverage, + ilikeRank, + RELATIVE_ILIKE_MAX_BOOST, + bestVector, + ) + : ilikeRescueScore( + Number(row.match_count), + words.length, + ilikeRank, + ); ftsResultIds.add(row.id); keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { @@ -636,14 +678,37 @@ export class MemoryQueryService { select: { id: true }, }); let identityAdded = 0; - for (const row of identityResults) { + for ( + let identityRank = 0; + identityRank < identityResults.length; + identityRank++ + ) { + const row = identityResults[identityRank]; + // DEFAULT MODE: flat 1.15 — the same flat-constant defect the FTS + // and ILIKE paths had. 1.15 sits *inside* the FTS band + // (1.10, 1.25], so identity hits interleave with FTS hits and all + // ten identity rows tie with each other. Left unchanged here so + // the flag measures exactly one variable; the relative branch + // below is the fix. + const identityScore = this.relativeRescue + ? relativeRescueScore( + cosineMap.get(row.id), + 1, + identityRank, + RELATIVE_IDENTITY_MAX_BOOST, + bestVector, + ) + : 1.15; keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { - scoreMap.set(row.id, 1.15); + scoreMap.set(row.id, identityScore); memoryIds.push(row.id); identityAdded++; } else { - scoreMap.set(row.id, Math.max(scoreMap.get(row.id)!, 1.15)); + scoreMap.set( + row.id, + Math.max(scoreMap.get(row.id)!, identityScore), + ); } } if (identityAdded > 0) { @@ -701,6 +766,15 @@ export class MemoryQueryService { const mem = memoryMap.get(id); if (mem) keywordRescueMap.set(id, mem); } + // NOTE (research finding, unrelated to the flag): this loop is + // unreachable. `topIds` and `memoryMap` are both built from `sorted`, so + // `!topIds.has(id)` implies `memoryMap.get(id) === undefined` and the + // inner `if (mem)` never fires. `forcedFts` is therefore always empty and + // the flat `0.75` below is dead code — it is not a live scoring path. + // Deliberately left as-is: making it live would inject FTS-only + // candidates at a flat constant, i.e. reintroduce the exact defect + // c905438 removed. Fixing it properly means sourcing those memories from + // a second findMany, which is out of scope for this experiment. const forcedFts: MemoryWithScore[] = []; for (const id of ftsResultIds) { if (!topIds.has(id)) { @@ -792,15 +866,45 @@ export class MemoryQueryService { // to a flat 1.1 here would re-flatten exactly the scores the FTS/ILIKE // bands above made continuous; the band bottom is only a floor for the // pathological case of a rescued memory arriving with no score at all. + // + // RESEARCH FINDING — this is where the score scales get mixed. + // + // `applyReranking` rescales EVERY surviving candidate: with the + // cross-encoder it becomes `normalisedRerankScore * 0.85 + importance * + // 0.15` (≤ ~1.0); without it, the fallback blend is `(score * 0.85 + + // importance * 0.15) * importanceMultiplier * sentimentPenalty`. Either + // way the output lives well below 1.0. The candidates re-added below, + // however, carry their *pre-rerank* rescue score — 1.25 / 1.15 / 1.05 — + // straight from `keywordRescueMap`. Those raw band values are then sorted + // against rescaled values and win categorically, so the final top-N is + // whatever the lexical rescue produced and the reranker's judgement is + // discarded. It also makes results non-monotonic in `limit`: at limit=5 + // the page is 5 band hits, at limit=50 the same query returns pure + // reranked scores because nothing was "missing" to re-add. + const rescaledFloor = scoredMemories.reduce( + (min, m) => Math.min(min, m.score ?? 0), + Number.POSITIVE_INFINITY, + ); const missingKeywordHits = [...keywordRescueMap.entries()] .filter(([id]) => !scoredMemories.some((m) => m.id === id)) - .map( - ([, mem]) => - ({ + .map(([, mem], i) => { + if (!this.relativeRescue) { + return { ...mem, score: mem.score ?? ILIKE_RESCUE_BAND_BOTTOM, - }) as MemoryWithScore, - ); + } as MemoryWithScore; + } + // Relative mode: honour the "sticky" intent (an exact-match write must + // not vanish from the result set) without letting a pre-rerank score + // outrank post-rerank ones. Re-added hits are appended just below the + // lowest rescaled score, ordered among themselves by their own rescue + // score. + const floor = Number.isFinite(rescaledFloor) ? rescaledFloor : 0; + return { + ...mem, + score: Math.min(mem.score ?? 0, floor) - 1e-9 * (i + 1), + } as MemoryWithScore; + }); if (missingKeywordHits.length > 0) { scoredMemories = [...missingKeywordHits, ...scoredMemories] .sort(compareByRankKeys) diff --git a/src/memory/memory-ranking.util.spec.ts b/src/memory/memory-ranking.util.spec.ts index 6a7c4ff6..8c8ead9d 100644 --- a/src/memory/memory-ranking.util.spec.ts +++ b/src/memory/memory-ranking.util.spec.ts @@ -9,6 +9,10 @@ import { ILIKE_RESCUE_BAND_TOP, ILIKE_RESCUE_BAND_BOTTOM, RankableMemory, + relativeRescueScore, + RELATIVE_FTS_MAX_BOOST, + RELATIVE_ILIKE_MAX_BOOST, + RELATIVE_LEXICAL_ONLY_ANCHOR, } from './memory-ranking.util'; describe('memory-ranking.util', () => { @@ -214,4 +218,98 @@ describe('memory-ranking.util', () => { expect(compareByRankKeys(x, x)).toBe(0); }); }); + + // PROTOTYPE (RECALL_RELATIVE_RESCUE=true). Default behaviour is unaffected. + describe('relativeRescueScore', () => { + it('never exceeds the cosine ceiling by more than the boost', () => { + const s = relativeRescueScore(1.0, 1, 0, RELATIVE_FTS_MAX_BOOST, 1.0); + expect(s).toBeLessThanOrEqual(1 + RELATIVE_FTS_MAX_BOOST + 1e-6); + }); + + it('is monotonically increasing in cosine at equal lexical quality', () => { + const weakSemantic = relativeRescueScore( + 0.4, + 1, + 0, + RELATIVE_FTS_MAX_BOOST, + 0.9, + ); + const strongSemantic = relativeRescueScore( + 0.9, + 1, + 5, + RELATIVE_FTS_MAX_BOOST, + 0.9, + ); + // This is the property the band inverts: under the band the rank-0 + // lexical hit wins regardless of cosine. + expect(strongSemantic).toBeGreaterThan(weakSemantic); + }); + + it('lets a strong semantic hit beat a perfect lexical hit on a weak one', () => { + // Best-possible FTS agreement on a 0.60-cosine candidate... + const lexicalWinnerOnWeakDoc = relativeRescueScore( + 0.6, + 1, + 0, + RELATIVE_FTS_MAX_BOOST, + 0.95, + ); + // ...still loses to a plain 0.95-cosine candidate with no lexical hit. + expect(lexicalWinnerOnWeakDoc).toBeLessThan(0.95); + }); + + it('still promotes: a lexical hit outranks the same cosine without one', () => { + const cosine = 0.7; + const boosted = relativeRescueScore( + cosine, + 1, + 0, + RELATIVE_ILIKE_MAX_BOOST, + 0.9, + ); + expect(boosted).toBeGreaterThan(cosine); + expect(boosted).toBeCloseTo(cosine * (1 + RELATIVE_ILIKE_MAX_BOOST), 5); + }); + + it('anchors lexical-only candidates below the best semantic hit', () => { + const bestVector = 0.82; + const lexicalOnly = relativeRescueScore( + null, + 1, + 0, + RELATIVE_FTS_MAX_BOOST, + bestVector, + ); + expect(lexicalOnly).toBeLessThan(bestVector); + expect(lexicalOnly).toBeCloseTo( + bestVector * RELATIVE_LEXICAL_ONLY_ANCHOR, + 5, + ); + }); + + it('breaks exact ties strictly by lexical rank', () => { + const a = relativeRescueScore(0.5, 1, 0, RELATIVE_FTS_MAX_BOOST, 0.5); + const b = relativeRescueScore(0.5, 1, 1, RELATIVE_FTS_MAX_BOOST, 0.5); + expect(a).toBeGreaterThan(b); + }); + + it('scales lexical quality: full coverage beats partial at equal cosine', () => { + const full = relativeRescueScore( + 0.6, + 1, + 0, + RELATIVE_ILIKE_MAX_BOOST, + 0.9, + ); + const partial = relativeRescueScore( + 0.6, + 0.33, + 0, + RELATIVE_ILIKE_MAX_BOOST, + 0.9, + ); + expect(full).toBeGreaterThan(partial); + }); + }); }); diff --git a/src/memory/memory-ranking.util.ts b/src/memory/memory-ranking.util.ts index 6b93bbe3..bae37389 100644 --- a/src/memory/memory-ranking.util.ts +++ b/src/memory/memory-ranking.util.ts @@ -122,6 +122,86 @@ export function ilikeRescueScore( ); } +/* ─────────────────────────────────────────────────────────────────────────── + * PROTOTYPE (feature-flagged, OFF by default): relative lexical rescue. + * + * See docs/research/memory-formation-query-transform/05-finding-band-inversion.md. + * + * The band design above places every lexical-rescue hit ABOVE the cosine + * ceiling of 1.0. That makes the priority *categorical*: a weak lexical match + * always outranks a strong semantic match, and the candidate's own cosine is + * discarded as an ordering signal even when it is available. On a corpus whose + * distractors restate the query verbatim, the entire band is distractors. + * + * Relative mode instead treats lexical agreement as a *bounded boost on the + * semantic score* rather than a separate stratum: + * + * score = cosine * (1 + maxBoost * quality) + * + * Consequences: + * - a lexical hit can promote a candidate, but a candidate with a much better + * cosine can still beat it (the boost is bounded by `maxBoost`); + * - the cosine ordering inside the rescued set is preserved instead of being + * overwritten by ts_rank, which is what the band does today; + * - all scores stay in the same numeric scale, so downstream stages + * (usage weighting, the fallback blend, the cross-encoder) compare like + * with like. + * + * Lexical-only candidates (matched by FTS/ILIKE but absent from the vector + * pool) have no cosine to anchor to. They are admitted just below the best + * observed cosine, so they enter the candidate pool but cannot displace the + * best semantic hit. + * ────────────────────────────────────────────────────────────────────────── */ + +/** Maximum multiplicative promotion an FTS/BM25 agreement may buy. */ +export const RELATIVE_FTS_MAX_BOOST = 0.2; +/** Maximum multiplicative promotion an ILIKE coverage agreement may buy. */ +export const RELATIVE_ILIKE_MAX_BOOST = 0.1; +/** Maximum multiplicative promotion the identity-profile rescue may buy. */ +export const RELATIVE_IDENTITY_MAX_BOOST = 0.15; +/** + * A lexical-only candidate (no vector hit) is anchored at this fraction of the + * best observed cosine, so it can never outrank the best semantic hit. + */ +export const RELATIVE_LEXICAL_ONLY_ANCHOR = 0.9; + +/** + * Relative-mode rescue score. + * + * @param cosine the candidate's own cosine similarity, or null/undefined if + * it was not in the vector pool + * @param quality normalised lexical agreement in [0, 1] (ts_rank / max + * ts_rank for FTS, matched/total terms for ILIKE) + * @param rankIndex position in the lexical result set, used only as a strict + * tie-break so two candidates never collide + * @param maxBoost the ceiling on the promotion this signal may buy + * @param bestVector the best cosine observed in this query's vector pool, + * used to anchor lexical-only candidates + */ +export function relativeRescueScore( + cosine: number | null | undefined, + quality: number, + rankIndex: number, + maxBoost: number, + bestVector: number, +): number { + const q = Number.isFinite(quality) ? Math.min(1, Math.max(0, quality)) : 0; + // Positional term is deliberately tiny: it exists only to break exact ties. + const positional = 1e-6 * rrfNorm(rankIndex); + + if (cosine != null && Number.isFinite(cosine) && cosine > 0) { + return cosine * (1 + maxBoost * q) + positional; + } + + // Lexical-only: no semantic evidence at all. Anchor strictly below the best + // semantic hit so it is admitted to the pool but cannot win on lexical + // agreement alone. + const anchor = + (Number.isFinite(bestVector) && bestVector > 0 ? bestVector : 0.5) * + RELATIVE_LEXICAL_ONLY_ANCHOR; + return anchor * (0.5 + 0.5 * q) + positional; +} + /** Minimal shape the comparator needs. Deliberately structural, not `Memory`. */ export interface RankableMemory { id?: string; From 7cdf49d91a9874e92727f7eabf0caeeb3576cb82 Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 18:22:05 -0700 Subject: [PATCH 6/8] research(recall): fix rerank/re-add scale mixing behind RECALL_RERANK_SCALE_FIX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sticky keyword re-add reinjected rescued candidates at their raw pre-rerank band scores (1.25 / 1.15 / 1.05) into an ordering that applyReranking had already rescaled to <= ~1.0. Raw beat rescaled categorically, so the limit=10 page was whatever the lexical rescue produced and the reranker's judgement was discarded — and ranking was non-monotonic in `limit` (the same memory scored 1.24804 at limit=10 and 0.42623 at limit=250). RECALL_RERANK_SCALE_FIX=true (default off) ranks the whole candidate pool instead of truncating at `limit`, so rescued candidates stay sticky *in the rescaled scale* and the re-add becomes a no-op; `limit` is applied once, at the end, to a total order. Adds RECALL_NO_RESCUE=true as the vector-only control arm. Measured on the noisy mnemon corpus (20 tasks, limit=10, usage counters reset per arm): arm gold@1 gold@5 gold@10 MRR@10 default 0 6 6 0.1500 scalefix 0 14 14 0.3500 relative 0 7 8 0.1738 both 0 7 8 0.1738 norescue 6 7 8 0.3312 Limit-monotonicity verified live: default violates the prefix property on 20/20 queries, scalefix holds on 20/20 (both 5-vs-10 same-pool and 10-vs-250). Default arm output is unchanged with all flags off. Co-Authored-By: Claude Opus 4.7 --- scripts/research/check-limit-monotonicity.mjs | 81 +++++ scripts/research/run-scale-fix-arms.sh | 75 +++++ scripts/research/score-arms.mjs | 50 ++++ src/memory/memory-query-scale-fix.spec.ts | 281 ++++++++++++++++++ src/memory/memory-query.service.ts | 114 +++++-- 5 files changed, 575 insertions(+), 26 deletions(-) create mode 100644 scripts/research/check-limit-monotonicity.mjs create mode 100755 scripts/research/run-scale-fix-arms.sh create mode 100755 scripts/research/score-arms.mjs create mode 100644 src/memory/memory-query-scale-fix.spec.ts diff --git a/scripts/research/check-limit-monotonicity.mjs b/scripts/research/check-limit-monotonicity.mjs new file mode 100644 index 00000000..3633a708 --- /dev/null +++ b/scripts/research/check-limit-monotonicity.mjs @@ -0,0 +1,81 @@ +#!/usr/bin/env node +/** + * Live check of the limit-monotonicity property against a running Engram. + * + * Property: for the same candidate pool, the top-K page of a `limit=K` query + * must be the prefix of the top-K of a `limit=N` query (N > K). + * + * Two comparisons are reported, because the recall path has TWO sources of + * limit-dependence and only one of them is the ranking defect: + * + * 5 vs 10 — `candidateLimit = max(200, limit*20)` is 200 for both, so the + * candidate pool is identical. Any violation here is pure + * ranking policy. This is the property the fix targets. + * 10 vs 250 — the deep call widens the vector pool (200 → up to 5000), so a + * violation here can also be caused by candidates the shallow + * call never saw. Reported for information. + * + * Usage: PROBE_API_KEY=... node scripts/research/check-limit-monotonicity.mjs + */ +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; + +const MNEMON = process.env.PROBE_MNEMON ?? '/Users/beauxwalton/projects/mnemon'; +const BASE = (process.env.PROBE_BASE_URL ?? 'http://127.0.0.1:3007').replace(/\/+$/, ''); +const KEY = process.env.PROBE_API_KEY; +const USER = process.env.PROBE_USER_ID ?? 'mnemon-v02-noise-prefix'; +const TASKS = process.env.PROBE_TASKS_DIR ?? join(MNEMON, '.mnemon/noisy-tasks-prefix'); + +if (!KEY) { + process.stderr.write('PROBE_API_KEY is required\n'); + process.exit(2); +} + +const label = process.argv[2] ?? 'unlabelled'; + +async function recall(query, limit) { + const res = await fetch(`${BASE}/v1/memories/query?scope=user`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-AM-API-Key': KEY, + 'X-AM-User-ID': USER, + }, + body: JSON.stringify({ query, limit }), + }); + if (!res.ok) throw new Error(`recall ${res.status}: ${(await res.text()).slice(0, 300)}`); + return (await res.json()).memories ?? []; +} + +const queries = readdirSync(TASKS) + .filter((f) => /^\d\d-.*\.json$/.test(f)) + .sort() + .map((f) => JSON.parse(readFileSync(join(TASKS, f), 'utf8'))); + +let samePoolViolations = 0; +let widePoolViolations = 0; + +for (const t of queries) { + // NOTE: recall increments retrieval_count, which feeds usage weighting. Issue + // the WIDER call first so the narrow page is not scored against counters the + // wide call just moved. + const deep = (await recall(t.query, 250)).map((m) => m.id); + const ten = (await recall(t.query, 10)).map((m) => m.id); + const five = (await recall(t.query, 5)).map((m) => m.id); + + const samePool = five.join() === ten.slice(0, five.length).join(); + const widePool = ten.join() === deep.slice(0, ten.length).join(); + if (!samePool) { + samePoolViolations++; + process.stdout.write(` VIOLATION (5 vs 10) ${t.id}\n 5: ${five.join(' ')}\n 10: ${ten.slice(0, 5).join(' ')}\n`); + } + if (!widePool) widePoolViolations++; +} + +process.stdout.write( + `\n[${label}] limit-monotonicity over ${queries.length} queries\n` + + ` same pool (5 vs 10): ${queries.length - samePoolViolations}/${queries.length} hold\n` + + ` wider pool (10 vs 250): ${queries.length - widePoolViolations}/${queries.length} hold\n`, +); + +process.exit(samePoolViolations === 0 ? 0 : 1); diff --git a/scripts/research/run-scale-fix-arms.sh b/scripts/research/run-scale-fix-arms.sh new file mode 100755 index 00000000..5ad1382b --- /dev/null +++ b/scripts/research/run-scale-fix-arms.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Five-arm retrieval A/B on the noisy mnemon corpus. +# +# default — shipped behaviour (banded rescue + raw-score sticky re-add) +# scalefix — RECALL_RERANK_SCALE_FIX=true (rank full pool, no raw re-add) +# relative — RECALL_RELATIVE_RESCUE=true (prior prototype) +# both — scalefix + relative +# norescue — RECALL_NO_RESCUE=true (vector-only control) +# +# Retrieval-only. No LLM, no generation. Restarts the research Engram instance +# once per arm and resets usage counters (retrieval_count / used_count / +# last_retrieved_at) between arms, because recall increments them and +# usage-weighted reranking feeds them back at weight 0.15 — without the reset +# arm N is measured against a system arm N-1 mutated. +# +# Usage: scripts/research/run-scale-fix-arms.sh +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KEY="${1:-${PROBE_API_KEY:-}}" +DB="${PROBE_DB_URL:-postgresql://engram:engram@localhost:5432/engram_ablation_prefix}" +PORT="${PROBE_PORT:-3007}" + +[ -n "$KEY" ] || { echo "usage: $0 " >&2; exit 2; } + +reset_usage() { + psql "$DB" -q -c " + UPDATE memories m + SET retrieval_count = s.retrieval_count, + used_count = s.used_count, + unused_count = s.unused_count, + last_retrieved_at = s.last_retrieved_at, + last_used_at = s.last_used_at + FROM zz_probe_usage_snapshot s + WHERE m.id = s.id;" >/dev/null +} + +start_server() { + local arm="$1" scalefix="$2" relative="$3" norescue="$4" + pkill -f "node dist/main.js" 2>/dev/null || true + sleep 2 + cd "$REPO" + RECALL_RERANK_SCALE_FIX="$scalefix" \ + RECALL_RELATIVE_RESCUE="$relative" \ + RECALL_NO_RESCUE="$norescue" \ + nohup node dist/main.js >"/tmp/engram-arm-${arm}.log" 2>&1 & + for _ in $(seq 1 60); do + if curl -sf -m 2 "http://127.0.0.1:${PORT}/v1/health" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + echo "server did not come up (arm=$arm)" >&2; exit 1 +} + +# arm : scalefix : relative : norescue +ARMS=( + "default:false:false:false" + "scalefix:true:false:false" + "relative:false:true:false" + "both:true:true:false" + "norescue:false:false:true" +) + +for SPEC in "${ARMS[@]}"; do + IFS=: read -r ARM SF REL NR <<<"$SPEC" + echo "### arm=$ARM scalefix=$SF relative=$REL norescue=$NR" + start_server "$ARM" "$SF" "$REL" "$NR" + reset_usage + PROBE_API_KEY="$KEY" node "$REPO/scripts/research/retrieval-probe.mjs" \ + --limit 10 --label "$ARM" --out "/tmp/arm-${ARM}.json" \ + | tail -12 +done + +node "$REPO/scripts/research/score-arms.mjs" \ + /tmp/arm-default.json /tmp/arm-scalefix.json /tmp/arm-relative.json \ + /tmp/arm-both.json /tmp/arm-norescue.json diff --git a/scripts/research/score-arms.mjs b/scripts/research/score-arms.mjs new file mode 100755 index 00000000..a4c3bdbd --- /dev/null +++ b/scripts/research/score-arms.mjs @@ -0,0 +1,50 @@ +#!/usr/bin/env node +/** + * Summarise retrieval-probe artifacts into a comparison table. + * + * gold@1 / @5 / @10 are counts of tasks whose gold memory landed in the + * limit=10 page at that depth. MRR@10 is the mean of 1/(rank+1) over all tasks, + * counting 0 for tasks where gold never appears in the page — so it is + * comparable across arms with different miss rates. + * + * poolMedian is the gold's median rank in the deep (limit=250) recall for that + * arm: the ordering the page *could* have delivered. + * + * Usage: node scripts/research/score-arms.mjs [...] + */ +import { readFileSync } from 'node:fs'; + +const files = process.argv.slice(2); +if (files.length === 0) { + process.stderr.write('usage: score-arms.mjs [...]\n'); + process.exit(2); +} + +function median(xs) { + if (xs.length === 0) return NaN; + const s = [...xs].sort((a, b) => a - b); + const m = s.length >> 1; + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} + +const rows = []; +for (const f of files) { + const { summary, rows: tasks } = JSON.parse(readFileSync(f, 'utf8')); + const n = tasks.length; + const at = (k) => tasks.filter((t) => t.rankInPage >= 0 && t.rankInPage < k).length; + const mrr = + tasks.reduce((a, t) => a + (t.rankInPage >= 0 ? 1 / (t.rankInPage + 1) : 0), 0) / n; + rows.push({ + arm: summary.label, + n, + 'gold@1': at(1), + 'gold@5': at(5), + 'gold@10': at(10), + 'MRR@10': Number(mrr.toFixed(4)), + poolMedian: median(tasks.map((t) => t.rankInPool)), + poolTop5: tasks.filter((t) => t.rankInPool >= 0 && t.rankInPool < 5).length, + inPool: tasks.filter((t) => t.rankInPool >= 0).length, + }); +} + +console.table(rows); diff --git a/src/memory/memory-query-scale-fix.spec.ts b/src/memory/memory-query-scale-fix.spec.ts new file mode 100644 index 00000000..e1b16cd5 --- /dev/null +++ b/src/memory/memory-query-scale-fix.spec.ts @@ -0,0 +1,281 @@ +/** + * Research: RECALL_RERANK_SCALE_FIX / RECALL_NO_RESCUE. + * + * The defect under test: `applyReranking` rescales every candidate to a + * post-rerank scale (≤ ~1.0), and the sticky keyword re-add then re-injects + * rescued candidates at their RAW pre-rerank band scores (1.25 / 1.15 / 1.05). + * Raw values are sorted against rescaled ones, the band always wins, and the + * page therefore depends on `limit` in a non-monotonic way: at `limit=K` the + * page is band hits, at `limit=poolSize` the same query returns a purely + * reranked ordering because nothing was "missing" to re-add. + * + * These tests pin both the defect (flag off — shipped behaviour must not move) + * and the invariant the fix buys (flag on — prefix property in `limit`). + */ +import { MemoryQueryService } from './memory-query.service'; +import { MemoryQueryRankingService } from './memory-query-ranking.service'; +import { MemoryQueryContextService } from './memory-query-context.service'; +import { PrismaService } from '../prisma/prisma.service'; +import { EmbeddingService } from './embedding.service'; +import { TemporalParserService } from './temporal/temporal-parser.service'; +import { RecallWeightService } from './recall-weight.service'; + +const QUERY = 'benchmark project fact user creation endpoint'; +const POOL_SIZE = 40; + +/** Candidate i: descending cosine, so the pure-semantic order is m0, m1, … */ +function candidateId(i: number): string { + return `m${String(i).padStart(3, '0')}`; +} + +interface Harness { + service: MemoryQueryService; + prisma: any; +} + +/** + * Stand-in for the TEI cross-encoder. Scores by semantic position (candidate 0 + * best), which is the regime the production reranker is in: it overwrites the + * incoming score entirely, so lexical band values never reach the output scale. + */ +function fakeReranker(): any { + return { + rerank: (_q: string, texts: string[]) => + Promise.resolve( + texts.map((t, index) => ({ + index, + score: 1 - Number(/^memory (\d+)/.exec(t)?.[1] ?? index) * 0.01, + })), + ), + }; +} + +/** + * Builds a service over a fixed 40-candidate pool. + * + * The lexically-rescued candidates are deliberately the WORST semantic ones + * (the tail of the pool), which is the situation the corpus actually exhibits: + * off-topic distractors that happen to restate the query's vocabulary. + */ +function harness(opts: { reranker?: boolean } = {}): Harness { + const memories = Array.from({ length: POOL_SIZE }, (_, i) => ({ + id: candidateId(i), + raw: `memory ${i} about ${QUERY}`, + userId: 'user-1', + importanceScore: 0.5, + effectiveScore: 0.5, + createdAt: new Date(2026, 0, 1 + i), + extraction: {}, + })); + + // Bottom 10 of the pool are the lexical "rescue" hits. + const rescuedIds = memories.slice(POOL_SIZE - 10).map((m) => m.id); + + const prisma = { + memory: { + findMany: jest.fn().mockResolvedValue(memories), + updateMany: jest.fn().mockResolvedValue({ count: 0 }), + }, + // First call is FTS (returns {id, ts_rank}); later calls are the ILIKE + // rescue ({id, match_count}). Returning FTS rows for both is fine — the + // ILIKE loop only reads `match_count`, which is undefined → clamped. + $queryRawUnsafe: jest.fn().mockImplementation(() => + Promise.resolve( + rescuedIds.map((id, i) => ({ + id, + ts_rank: 1 - i * 0.01, + match_count: 3, + })), + ), + ), + } as any as PrismaService; + + const embedding = { + generate: jest.fn().mockResolvedValue([0.1, 0.2, 0.3]), + generateForRecall: jest.fn().mockResolvedValue([0.1, 0.2, 0.3]), + search: jest + .fn() + .mockImplementation(() => + Promise.resolve( + memories.map((m, i) => ({ id: m.id, score: 0.9 - i * 0.01 })), + ), + ), + } as any as EmbeddingService; + + const temporalParser = { + parse: jest.fn().mockReturnValue({ + semanticQuery: QUERY, + temporalFilter: null, + }), + } as any as TemporalParserService; + + const recallWeightService = { + recallWeight: jest.fn().mockReturnValue(1.0), + applyUsageWeighting: jest + .fn() + .mockImplementation((mems: any[]) => Promise.resolve(mems)), + } as any as RecallWeightService; + + const rankingService = new MemoryQueryRankingService( + prisma, + embedding, + recallWeightService, + opts.reranker ? fakeReranker() : undefined, + ); + // Insight surfacing / graph merge are separate stages with their own + // `limit`-dependent fetches; neutralise them so this test isolates the + // rerank ↔ re-add interaction. + jest + .spyOn(rankingService, 'surfaceInsights') + .mockImplementation((existing: any) => Promise.resolve(existing)); + + const contextService = new MemoryQueryContextService(prisma); + + const service = new MemoryQueryService( + prisma, + embedding, + temporalParser, + recallWeightService, + rankingService, + contextService, + ); + + return { service, prisma }; +} + +async function page( + service: MemoryQueryService, + limit: number, +): Promise> { + const res = await service.recall('user-1', { query: QUERY, limit } as any); + return res.memories.map((m) => ({ id: m.id, score: m.score as number })); +} + +async function pageIds( + service: MemoryQueryService, + limit: number, +): Promise { + return (await page(service, limit)).map((m) => m.id); +} + +/** + * Scale consistency: a memory's score must not depend on the `limit` it was + * asked for. Any difference means two different scales were emitted for the + * same candidate — which is exactly the raw-band re-injection defect. + */ +async function scaleDrift( + service: MemoryQueryService, + limit: number, +): Promise { + const shallow = await page(service, limit); + const deep = new Map((await page(service, POOL_SIZE)).map((m) => [m.id, m])); + let worst = 0; + for (const m of shallow) { + const d = deep.get(m.id); + if (!d) continue; + worst = Math.max(worst, Math.abs(m.score - d.score)); + } + return worst; +} + +describe('recall ranking: limit-monotonicity (RECALL_RERANK_SCALE_FIX)', () => { + const savedEnv = { ...process.env }; + + afterEach(() => { + process.env = { ...savedEnv }; + jest.restoreAllMocks(); + }); + + describe('default (all research flags off) — shipped behaviour', () => { + beforeEach(() => { + delete process.env.RECALL_RERANK_SCALE_FIX; + delete process.env.RECALL_RELATIVE_RESCUE; + delete process.env.RECALL_NO_RESCUE; + }); + + it('emits two different scales for the same memory (the defect)', async () => { + const { service } = harness({ reranker: true }); + // Raw band (~1.05–1.25) in the small page vs rescaled (<1) when deep. + expect(await scaleDrift(service, 5)).toBeGreaterThan(0.1); + }); + + it('is NOT monotonic in limit (documents the defect)', async () => { + const { service } = harness({ reranker: true }); + const shallow = await pageIds(service, 5); + const deep = await pageIds(service, POOL_SIZE); + expect(shallow).not.toEqual(deep.slice(0, shallow.length)); + // The small page is the lexical tail — the WORST semantic candidates — + // re-added at raw band scores that outrank every rescaled score. + expect(shallow.every((id) => Number(id.slice(1)) >= POOL_SIZE - 10)).toBe( + true, + ); + }); + }); + + describe('RECALL_RERANK_SCALE_FIX=true', () => { + beforeEach(() => { + process.env.RECALL_RERANK_SCALE_FIX = 'true'; + delete process.env.RECALL_RELATIVE_RESCUE; + delete process.env.RECALL_NO_RESCUE; + }); + + it.each([ + ['with cross-encoder', true], + ['fallback blend only', false], + ])( + 'top-K of limit=K is the prefix of the deep ranking (%s)', + async (_label, reranker) => { + const { service } = harness({ reranker }); + const deep = await pageIds(service, POOL_SIZE); + + for (const k of [1, 3, 5, 10, 20]) { + const ids = await pageIds(service, k); + expect(ids).toHaveLength(k); + expect(ids).toEqual(deep.slice(0, k)); + } + }, + ); + + it('emits one consistent scale: a memory scores the same at any limit', async () => { + const { service } = harness({ reranker: true }); + expect(await scaleDrift(service, 5)).toBeLessThan(1e-9); + expect(await scaleDrift(service, 10)).toBeLessThan(1e-9); + }); + + it('puts the best semantic candidates on the page, not the lexical tail', async () => { + const { service } = harness({ reranker: true }); + expect(await pageIds(service, 5)).toEqual( + [0, 1, 2, 3, 4].map(candidateId), + ); + }); + + it('still keeps rescued candidates in the pool (sticky, just not raw)', async () => { + const { service } = harness({ reranker: true }); + const deep = await pageIds(service, POOL_SIZE); + expect(deep).toHaveLength(POOL_SIZE); + for (let i = POOL_SIZE - 10; i < POOL_SIZE; i++) { + expect(deep).toContain(candidateId(i)); + } + }); + }); + + describe('RECALL_NO_RESCUE=true (vector-only control)', () => { + beforeEach(() => { + process.env.RECALL_NO_RESCUE = 'true'; + delete process.env.RECALL_RERANK_SCALE_FIX; + delete process.env.RECALL_RELATIVE_RESCUE; + }); + + it('issues no lexical rescue SQL at all', async () => { + const { service, prisma } = harness(); + await pageIds(service, 10); + expect(prisma.$queryRawUnsafe).not.toHaveBeenCalled(); + }); + + it('returns the pure semantic ordering', async () => { + const { service } = harness(); + const page = await pageIds(service, 5); + expect(page).toEqual([0, 1, 2, 3, 4].map(candidateId)); + }); + }); +}); diff --git a/src/memory/memory-query.service.ts b/src/memory/memory-query.service.ts index 42ef1f8d..9daa468b 100644 --- a/src/memory/memory-query.service.ts +++ b/src/memory/memory-query.service.ts @@ -70,6 +70,40 @@ export class MemoryQueryService { private readonly relativeRescue = process.env.RECALL_RELATIVE_RESCUE === 'true'; + /** + * RESEARCH FLAG — default OFF, no change to shipped behaviour. + * + * `RECALL_RERANK_SCALE_FIX=true` removes the score-scale mixing between the + * reranker and the sticky keyword re-add (see the long comment at the re-add + * site below). Two coupled changes, both required for the invariant: + * + * 1. `applyReranking` is asked to rank the WHOLE candidate pool instead of + * truncating at `limit`. Every keyword-rescued candidate therefore + * survives reranking *in the rescaled scale*, so "sticky" is satisfied + * structurally rather than by re-injecting a pre-rerank score. + * 2. Anything that still has to be re-added (defensive path only, e.g. a + * candidate dropped by a downstream stage) is placed just below the + * lowest rescaled score instead of at its raw 1.25 / 1.15 / 1.05 band + * value, so raw and rescaled values are never sorted against each other. + * + * Consequence — the property the old code violated: ranking becomes + * MONOTONIC IN `limit`. The top-K of a `limit=K` query is exactly the prefix + * of the top-K of a `limit=N` query (N > K) for the same candidate pool. + */ + private readonly rerankScaleFix = + process.env.RECALL_RERANK_SCALE_FIX === 'true'; + + /** + * RESEARCH FLAG — default OFF. Control arm. + * + * `RECALL_NO_RESCUE=true` disables every lexical rescue path (FTS/BM25, + * ILIKE coverage, identity profile) so recall is vector-only. This is the + * honest baseline for "what can the embedding do on its own", against which + * any rescue policy — banded, relative, or scale-fixed — has to justify + * itself. + */ + private readonly noRescue = process.env.RECALL_NO_RESCUE === 'true'; + constructor( private prisma: PrismaService, private embedding: EmbeddingService, @@ -419,9 +453,11 @@ export class MemoryQueryService { : []; try { type FtsRow = { id: string; ts_rank: number }; - const ftsResults = poolOnlyMode - ? await this.prisma.$queryRawUnsafe( - `SELECT m.id, + const ftsResults = this.noRescue + ? [] + : poolOnlyMode + ? await this.prisma.$queryRawUnsafe( + `SELECT m.id, ts_rank(to_tsvector('english', m.raw), websearch_to_tsquery('english', $2)) AS ts_rank FROM memories m WHERE m.id IN ( @@ -436,12 +472,12 @@ export class MemoryQueryService { AND m.is_duplicate_of IS NULL ORDER BY ts_rank(to_tsvector('english', m.raw), websearch_to_tsquery('english', $2)) DESC LIMIT 100`, - poolIds, - searchQuery, - ) - : resolvedUserIds.length > 0 - ? await this.prisma.$queryRawUnsafe( - `SELECT id, + poolIds, + searchQuery, + ) + : resolvedUserIds.length > 0 + ? await this.prisma.$queryRawUnsafe( + `SELECT id, ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $2)) AS ts_rank FROM memories WHERE user_id = ANY($1::text[]) @@ -453,11 +489,11 @@ export class MemoryQueryService { AND is_duplicate_of IS NULL ORDER BY ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $2)) DESC LIMIT 100`, - resolvedUserIds, - searchQuery, - ) - : await this.prisma.$queryRawUnsafe( - `SELECT id, + resolvedUserIds, + searchQuery, + ) + : await this.prisma.$queryRawUnsafe( + `SELECT id, ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $1)) AS ts_rank FROM memories WHERE to_tsvector('english', raw) @@ websearch_to_tsquery('english', $1) @@ -468,8 +504,8 @@ export class MemoryQueryService { AND is_duplicate_of IS NULL ORDER BY ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $1)) DESC LIMIT 100`, - searchQuery, - ); + searchQuery, + ); // Continuous FTS rescue band. Previously every FTS-rescued candidate // was assigned the constant 1.25, which is what made 18/20 benchmark // queries end in a 5-way tie at the top score. The score now varies @@ -522,7 +558,9 @@ export class MemoryQueryService { // websearch_to_tsquery can be too strict for natural questions (for // example requiring filler terms such as "tell" or "need"), while a // curated ILIKE pass catches exact domain words like medication/roast. - const words = this.extractLexicalRescueTerms(searchQuery); + const words = this.noRescue + ? [] + : this.extractLexicalRescueTerms(searchQuery); if (words.length > 0) { try { const hasResolvedUsers = resolvedUserIds.length > 0; @@ -649,7 +687,11 @@ export class MemoryQueryService { } } - if (this.isIdentityProfileQuery(searchQuery) && !poolOnlyMode) { + if ( + !this.noRescue && + this.isIdentityProfileQuery(searchQuery) && + !poolOnlyMode + ) { try { const identityResults = await this.prisma.memory.findMany({ where: { @@ -853,10 +895,17 @@ export class MemoryQueryService { // ── ENG-29: Cross-Encoder Reranking ────────────────────────── const rerankQuery = hasTemporalIntent ? dto.query : searchQuery; + // Scale-fix mode ranks the entire pool rather than truncating at `limit`. + // The reranker already scores every candidate (`candidates = memories`); + // `limit` only controlled the final `slice`. Ranking the whole pool is what + // makes the sticky re-add a no-op and the page prefix-stable in `limit`. + const rerankDepth = this.rerankScaleFix + ? Math.max(limit, scoredMemories.length) + : limit; scoredMemories = await this.rankingService.applyReranking( scoredMemories, rerankQuery, - limit, + rerankDepth, ); // Exact keyword/ILIKE rescued memories are deterministic high-signal hits. @@ -888,24 +937,37 @@ export class MemoryQueryService { const missingKeywordHits = [...keywordRescueMap.entries()] .filter(([id]) => !scoredMemories.some((m) => m.id === id)) .map(([, mem], i) => { - if (!this.relativeRescue) { + if (!this.relativeRescue && !this.rerankScaleFix) { return { ...mem, score: mem.score ?? ILIKE_RESCUE_BAND_BOTTOM, } as MemoryWithScore; } - // Relative mode: honour the "sticky" intent (an exact-match write must - // not vanish from the result set) without letting a pre-rerank score - // outrank post-rerank ones. Re-added hits are appended just below the - // lowest rescaled score, ordered among themselves by their own rescue - // score. + // Relative / scale-fix mode: honour the "sticky" intent (an exact-match + // write must not vanish from the result set) without letting a + // pre-rerank score outrank post-rerank ones. Re-added hits are appended + // just below the lowest rescaled score, ordered among themselves by + // their own rescue score. const floor = Number.isFinite(rescaledFloor) ? rescaledFloor : 0; return { ...mem, score: Math.min(mem.score ?? 0, floor) - 1e-9 * (i + 1), } as MemoryWithScore; }); - if (missingKeywordHits.length > 0) { + if (this.rerankScaleFix) { + // Always re-sort and truncate here: `applyReranking` was given the full + // pool depth above, so this is the single point where `limit` is applied. + // Truncating a total order (compareByRankKeys) at K is what gives the + // prefix property across different values of `limit`. + scoredMemories = ( + missingKeywordHits.length > 0 + ? [...missingKeywordHits, ...scoredMemories] + : scoredMemories + ) + .slice() + .sort(compareByRankKeys) + .slice(0, limit); + } else if (missingKeywordHits.length > 0) { scoredMemories = [...missingKeywordHits, ...scoredMemories] .sort(compareByRankKeys) .slice(0, limit); From 066b1e1720ffb34f652e37c7fca68a87e10a8c77 Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 19:08:35 -0700 Subject: [PATCH 7/8] =?UTF-8?q?research(recall):=20remove=20RECALL=5FRELAT?= =?UTF-8?q?IVE=5FRESCUE=20=E2=80=94=20it=20cancels=20the=20scale=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 5-arm ablation on the 20-task noisy prefix corpus (limit=10, usage counters reset per arm) killed the relative-rescue prototype: arm gold@1 gold@5 gold@10 MRR@10 default 0 6 6 0.150 scalefix 0 14 14 0.350 relative 0 7 8 0.174 both 0 7 8 0.174 vector-only 6 7 8 0.331 The kill signal is that `both` is numerically IDENTICAL to `relative`, not that `relative` merely underperformed. relativeRescueScore() rewrites candidate scores into the rescaled scale upstream, at the FTS/ILIKE/identity injection sites. By the time RECALL_RERANK_SCALE_FIX runs there is no scale mismatch left for it to correct, so the scale fix degrades to a no-op. The two flags do not compose: enabling relative rescue *replaces* the scale fix and drags 14/20 gold@5 back down to 7/20. They are mutually exclusive by construction, and the scale fix is strictly better on every metric. Leaving the prototype in the tree would have left a config that silently halves recall quality one env var away, so it goes now rather than rotting. Removed: - relativeRescueScore() and the RELATIVE_* boost/anchor constants - the RECALL_RELATIVE_RESCUE flag read and all three relative-mode branches (FTS, ILIKE, identity), plus the relative-mode variant of the sticky re-add's below-the-floor append - the 7 relativeRescueScore unit tests - scripts/research/run-rescue-ab.sh (existed only to A/B this flag); run-scale-fix-arms.sh drops the now-impossible relative/both arms Kept deliberately: - RECALL_RERANK_SCALE_FIX and RECALL_NO_RESCUE and everything they gate - everything from c905438 (ftsRescueScore, ilikeRescueScore, compareByRankKeys, the ts_rank SELECTs) - the unreachable-forcedFts annotation and the flat-1.15 identity note — findings, not prototype - all docs under docs/research/memory-formation-query-transform/. The analysis in 05-finding-band-inversion.md is still valid and is *why* we know to kill this; it now carries a note at the top recording the removal and the both==relative collapse so nobody re-derives it. memory-ranking.util.ts and memory-ranking.util.spec.ts are now byte-identical to c905438; the only remaining delta in memory-query.service.ts vs c905438 is the scale-fix work. With all flags off recall output is unchanged. src/memory: 67 suites / 1152 tests pass (1159 before; the 7 fewer are exactly the removed relativeRescueScore tests). Co-Authored-By: Claude Opus 4.7 --- .../05-finding-band-inversion.md | 35 ++++++ scripts/research/run-rescue-ab.sh | 54 --------- scripts/research/run-scale-fix-arms.sh | 33 +++--- src/memory/memory-query-scale-fix.spec.ts | 3 - src/memory/memory-query.service.ts | 108 ++++-------------- src/memory/memory-ranking.util.spec.ts | 98 ---------------- src/memory/memory-ranking.util.ts | 80 ------------- 7 files changed, 77 insertions(+), 334 deletions(-) delete mode 100755 scripts/research/run-rescue-ab.sh diff --git a/docs/research/memory-formation-query-transform/05-finding-band-inversion.md b/docs/research/memory-formation-query-transform/05-finding-band-inversion.md index 75ecca02..e7dff168 100644 --- a/docs/research/memory-formation-query-transform/05-finding-band-inversion.md +++ b/docs/research/memory-formation-query-transform/05-finding-band-inversion.md @@ -10,6 +10,41 @@ mnemon-v02-noise-prefix` --- +> ## ⚠️ UPDATE (2026-08-23, later the same day): the `RECALL_RELATIVE_RESCUE` prototype was REMOVED +> +> **The analysis in this document is still valid and is exactly why the prototype +> was killed. Only the prototype code is gone — `relativeRescueScore()`, the +> `RECALL_RELATIVE_RESCUE` flag and its branches, its unit tests, and +> `scripts/research/run-rescue-ab.sh`.** +> +> The 5-arm ablation that followed this write-up (20 tasks, noisy prefix corpus, +> `limit=10`, usage counters reset per arm) settled it: +> +> | arm | gold@1 | gold@5 | gold@10 | MRR@10 | +> | --- | --- | --- | --- | --- | +> | default | 0 | 6 | 6 | 0.150 | +> | scalefix (`RECALL_RERANK_SCALE_FIX`) | 0 | **14** | **14** | **0.350** | +> | relative (`RECALL_RELATIVE_RESCUE`) | 0 | 7 | 8 | 0.174 | +> | both | 0 | 7 | 8 | 0.174 | +> | vector-only (`RECALL_NO_RESCUE`) | 6 | 7 | 8 | 0.331 | +> +> The kill signal is that **`both` is numerically identical to `relative`**. +> `relativeRescueScore()` rewrites candidate scores into the rescaled scale +> *upstream*, so by the time `RECALL_RERANK_SCALE_FIX` runs there is no scale +> mismatch left for it to correct — the scale fix becomes a no-op. Enabling +> relative rescue therefore does not compose with the scale fix, it *replaces* +> it, dragging 14/20 gold@5 back down to 7/20. The two flags are mutually +> exclusive by construction, and the scale fix is strictly better on every +> metric. +> +> Keeping the prototype in the tree would only have tempted someone into a +> net-negative configuration. The sections below describe the removed code in the +> present tense; read them as a record of what was measured, not of what ships. +> The surviving fix is `RECALL_RERANK_SCALE_FIX` (commit `7cdf49d`, branch +> `fix/recall-rerank-scale-mixing`). + +--- + ## TL;DR 1. **Gold is never missing from the candidate pool.** 20/20 tasks retrieve the diff --git a/scripts/research/run-rescue-ab.sh b/scripts/research/run-rescue-ab.sh deleted file mode 100755 index 9dbc4f0b..00000000 --- a/scripts/research/run-rescue-ab.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# A/B the RECALL_RELATIVE_RESCUE prototype flag on the noisy mnemon corpus. -# -# Retrieval-only. No LLM, no generation. Restarts the research Engram instance -# once per arm and resets the usage counters (retrieval_count / used_count / -# last_retrieved_at) between arms, because recall increments them and -# usage-weighted reranking then feeds them back into the score — without the -# reset the second arm is measured against a different system than the first. -# -# Usage: scripts/research/run-rescue-ab.sh -set -euo pipefail - -REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -KEY="${1:-${PROBE_API_KEY:-}}" -DB="${PROBE_DB_URL:-postgresql://engram:engram@localhost:5432/engram_ablation_prefix}" -PORT="${PROBE_PORT:-3007}" - -[ -n "$KEY" ] || { echo "usage: $0 " >&2; exit 2; } - -reset_usage() { - psql "$DB" -q -c " - UPDATE memories m - SET retrieval_count = s.retrieval_count, - used_count = s.used_count, - unused_count = s.unused_count, - last_retrieved_at = s.last_retrieved_at, - last_used_at = s.last_used_at - FROM zz_probe_usage_snapshot s - WHERE m.id = s.id;" >/dev/null -} - -start_server() { - local mode="$1" - pkill -f "node dist/main.js" 2>/dev/null || true - sleep 2 - cd "$REPO" - RECALL_RELATIVE_RESCUE="$mode" nohup node dist/main.js \ - >"/tmp/engram-research-server-${mode}.log" 2>&1 & - for _ in $(seq 1 60); do - if curl -sf -m 2 "http://127.0.0.1:${PORT}/v1/health" >/dev/null 2>&1; then return 0; fi - sleep 1 - done - echo "server did not come up (mode=$mode)" >&2; exit 1 -} - -for MODE in false true; do - LABEL=$([ "$MODE" = "false" ] && echo BEFORE-band || echo AFTER-relative) - echo "### $LABEL (RECALL_RELATIVE_RESCUE=$MODE)" - start_server "$MODE" - reset_usage - PROBE_API_KEY="$KEY" node "$REPO/scripts/research/retrieval-probe.mjs" \ - --limit 10 --label "$LABEL" --out "/tmp/probe-${MODE}.json" \ - | tail -20 -done diff --git a/scripts/research/run-scale-fix-arms.sh b/scripts/research/run-scale-fix-arms.sh index 5ad1382b..48b2d442 100755 --- a/scripts/research/run-scale-fix-arms.sh +++ b/scripts/research/run-scale-fix-arms.sh @@ -1,12 +1,17 @@ #!/usr/bin/env bash -# Five-arm retrieval A/B on the noisy mnemon corpus. +# Three-arm retrieval A/B on the noisy mnemon corpus. # # default — shipped behaviour (banded rescue + raw-score sticky re-add) # scalefix — RECALL_RERANK_SCALE_FIX=true (rank full pool, no raw re-add) -# relative — RECALL_RELATIVE_RESCUE=true (prior prototype) -# both — scalefix + relative # norescue — RECALL_NO_RESCUE=true (vector-only control) # +# Originally five arms: the RECALL_RELATIVE_RESCUE prototype ("relative") and +# its combination with the scale fix ("both") have been removed along with the +# flag. `both` scored identically to `relative` (7/20 gold@5) because relative +# rescue rewrote candidate scores into the rescaled scale upstream, leaving the +# scale fix nothing to correct and dragging 14/20 back down to 7/20. See +# docs/research/memory-formation-query-transform/05-finding-band-inversion.md. +# # Retrieval-only. No LLM, no generation. Restarts the research Engram instance # once per arm and resets usage counters (retrieval_count / used_count / # last_retrieved_at) between arms, because recall increments them and @@ -36,12 +41,11 @@ reset_usage() { } start_server() { - local arm="$1" scalefix="$2" relative="$3" norescue="$4" + local arm="$1" scalefix="$2" norescue="$3" pkill -f "node dist/main.js" 2>/dev/null || true sleep 2 cd "$REPO" RECALL_RERANK_SCALE_FIX="$scalefix" \ - RECALL_RELATIVE_RESCUE="$relative" \ RECALL_NO_RESCUE="$norescue" \ nohup node dist/main.js >"/tmp/engram-arm-${arm}.log" 2>&1 & for _ in $(seq 1 60); do @@ -51,19 +55,17 @@ start_server() { echo "server did not come up (arm=$arm)" >&2; exit 1 } -# arm : scalefix : relative : norescue +# arm : scalefix : norescue ARMS=( - "default:false:false:false" - "scalefix:true:false:false" - "relative:false:true:false" - "both:true:true:false" - "norescue:false:false:true" + "default:false:false" + "scalefix:true:false" + "norescue:false:true" ) for SPEC in "${ARMS[@]}"; do - IFS=: read -r ARM SF REL NR <<<"$SPEC" - echo "### arm=$ARM scalefix=$SF relative=$REL norescue=$NR" - start_server "$ARM" "$SF" "$REL" "$NR" + IFS=: read -r ARM SF NR <<<"$SPEC" + echo "### arm=$ARM scalefix=$SF norescue=$NR" + start_server "$ARM" "$SF" "$NR" reset_usage PROBE_API_KEY="$KEY" node "$REPO/scripts/research/retrieval-probe.mjs" \ --limit 10 --label "$ARM" --out "/tmp/arm-${ARM}.json" \ @@ -71,5 +73,4 @@ for SPEC in "${ARMS[@]}"; do done node "$REPO/scripts/research/score-arms.mjs" \ - /tmp/arm-default.json /tmp/arm-scalefix.json /tmp/arm-relative.json \ - /tmp/arm-both.json /tmp/arm-norescue.json + /tmp/arm-default.json /tmp/arm-scalefix.json /tmp/arm-norescue.json diff --git a/src/memory/memory-query-scale-fix.spec.ts b/src/memory/memory-query-scale-fix.spec.ts index e1b16cd5..eb14b211 100644 --- a/src/memory/memory-query-scale-fix.spec.ts +++ b/src/memory/memory-query-scale-fix.spec.ts @@ -189,7 +189,6 @@ describe('recall ranking: limit-monotonicity (RECALL_RERANK_SCALE_FIX)', () => { describe('default (all research flags off) — shipped behaviour', () => { beforeEach(() => { delete process.env.RECALL_RERANK_SCALE_FIX; - delete process.env.RECALL_RELATIVE_RESCUE; delete process.env.RECALL_NO_RESCUE; }); @@ -215,7 +214,6 @@ describe('recall ranking: limit-monotonicity (RECALL_RERANK_SCALE_FIX)', () => { describe('RECALL_RERANK_SCALE_FIX=true', () => { beforeEach(() => { process.env.RECALL_RERANK_SCALE_FIX = 'true'; - delete process.env.RECALL_RELATIVE_RESCUE; delete process.env.RECALL_NO_RESCUE; }); @@ -263,7 +261,6 @@ describe('recall ranking: limit-monotonicity (RECALL_RERANK_SCALE_FIX)', () => { beforeEach(() => { process.env.RECALL_NO_RESCUE = 'true'; delete process.env.RECALL_RERANK_SCALE_FIX; - delete process.env.RECALL_RELATIVE_RESCUE; }); it('issues no lexical rescue SQL at all', async () => { diff --git a/src/memory/memory-query.service.ts b/src/memory/memory-query.service.ts index 9daa468b..6130c440 100644 --- a/src/memory/memory-query.service.ts +++ b/src/memory/memory-query.service.ts @@ -47,29 +47,12 @@ import { ftsRescueScore, ilikeRescueScore, ILIKE_RESCUE_BAND_BOTTOM, - relativeRescueScore, - RELATIVE_FTS_MAX_BOOST, - RELATIVE_ILIKE_MAX_BOOST, - RELATIVE_IDENTITY_MAX_BOOST, } from './memory-ranking.util'; @Injectable() export class MemoryQueryService { private readonly logger = new Logger(MemoryQueryService.name); - /** - * PROTOTYPE FLAG — default OFF, no change to shipped behaviour. - * - * `RECALL_RELATIVE_RESCUE=true` switches the keyword-rescue paths from the - * absolute score bands (FTS in (1.10,1.25], ILIKE in (1.00,1.10], identity a - * flat 1.15 — all of which sit above the cosine ceiling of 1.0 and therefore - * categorically outrank every semantic hit) to a bounded boost applied to the - * candidate's own cosine similarity. See - * docs/research/memory-formation-query-transform/05-finding-band-inversion.md. - */ - private readonly relativeRescue = - process.env.RECALL_RELATIVE_RESCUE === 'true'; - /** * RESEARCH FLAG — default OFF, no change to shipped behaviour. * @@ -518,26 +501,14 @@ export class MemoryQueryService { // Rows arrive ordered by ts_rank DESC, so row 0 carries the max. const maxTsRank = ftsResults.length > 0 ? Number(ftsResults[0].ts_rank) : 0; - // Relative mode only: the best cosine in this query's vector pool, used - // to anchor lexical-only candidates below the best semantic hit. - const bestVector = vectorResults.reduce( - (best, r) => (r.score > best ? r.score : best), - 0, - ); let ftsAdded = 0; for (let ftsRank = 0; ftsRank < ftsResults.length; ftsRank++) { const row = ftsResults[ftsRank]; - const quality = - maxTsRank > 0 ? Math.min(1, Number(row.ts_rank) / maxTsRank) : 1; - const bandedScore = this.relativeRescue - ? relativeRescueScore( - cosineMap.get(row.id), - quality, - ftsRank, - RELATIVE_FTS_MAX_BOOST, - bestVector, - ) - : ftsRescueScore(Number(row.ts_rank), maxTsRank, ftsRank); + const bandedScore = ftsRescueScore( + Number(row.ts_rank), + maxTsRank, + ftsRank, + ); ftsResultIds.add(row.id); keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { @@ -645,23 +616,11 @@ export class MemoryQueryService { ilikeRank++ ) { const row = ilikeResults[ilikeRank]; - const coverage = - words.length > 0 - ? Math.min(1, Number(row.match_count) / words.length) - : 1; - const bandedScore = this.relativeRescue - ? relativeRescueScore( - cosineMap.get(row.id), - coverage, - ilikeRank, - RELATIVE_ILIKE_MAX_BOOST, - bestVector, - ) - : ilikeRescueScore( - Number(row.match_count), - words.length, - ilikeRank, - ); + const bandedScore = ilikeRescueScore( + Number(row.match_count), + words.length, + ilikeRank, + ); ftsResultIds.add(row.id); keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { @@ -720,37 +679,20 @@ export class MemoryQueryService { select: { id: true }, }); let identityAdded = 0; - for ( - let identityRank = 0; - identityRank < identityResults.length; - identityRank++ - ) { - const row = identityResults[identityRank]; - // DEFAULT MODE: flat 1.15 — the same flat-constant defect the FTS - // and ILIKE paths had. 1.15 sits *inside* the FTS band - // (1.10, 1.25], so identity hits interleave with FTS hits and all - // ten identity rows tie with each other. Left unchanged here so - // the flag measures exactly one variable; the relative branch - // below is the fix. - const identityScore = this.relativeRescue - ? relativeRescueScore( - cosineMap.get(row.id), - 1, - identityRank, - RELATIVE_IDENTITY_MAX_BOOST, - bestVector, - ) - : 1.15; + // NOTE (research finding, not yet fixed): this flat 1.15 is the + // same flat-constant defect c905438 removed from the FTS and ILIKE + // paths. 1.15 sits *inside* the FTS band (1.10, 1.25], so identity + // hits interleave with FTS hits and all ten identity rows tie with + // each other. Left as-is deliberately — see + // docs/research/memory-formation-query-transform/05-finding-band-inversion.md. + for (const row of identityResults) { keywordRescueIds.add(row.id); if (!scoreMap.has(row.id)) { - scoreMap.set(row.id, identityScore); + scoreMap.set(row.id, 1.15); memoryIds.push(row.id); identityAdded++; } else { - scoreMap.set( - row.id, - Math.max(scoreMap.get(row.id)!, identityScore), - ); + scoreMap.set(row.id, Math.max(scoreMap.get(row.id)!, 1.15)); } } if (identityAdded > 0) { @@ -937,17 +879,17 @@ export class MemoryQueryService { const missingKeywordHits = [...keywordRescueMap.entries()] .filter(([id]) => !scoredMemories.some((m) => m.id === id)) .map(([, mem], i) => { - if (!this.relativeRescue && !this.rerankScaleFix) { + if (!this.rerankScaleFix) { return { ...mem, score: mem.score ?? ILIKE_RESCUE_BAND_BOTTOM, } as MemoryWithScore; } - // Relative / scale-fix mode: honour the "sticky" intent (an exact-match - // write must not vanish from the result set) without letting a - // pre-rerank score outrank post-rerank ones. Re-added hits are appended - // just below the lowest rescaled score, ordered among themselves by - // their own rescue score. + // Scale-fix mode: honour the "sticky" intent (an exact-match write must + // not vanish from the result set) without letting a pre-rerank score + // outrank post-rerank ones. Re-added hits are appended just below the + // lowest rescaled score, ordered among themselves by their own rescue + // score. const floor = Number.isFinite(rescaledFloor) ? rescaledFloor : 0; return { ...mem, diff --git a/src/memory/memory-ranking.util.spec.ts b/src/memory/memory-ranking.util.spec.ts index 8c8ead9d..6a7c4ff6 100644 --- a/src/memory/memory-ranking.util.spec.ts +++ b/src/memory/memory-ranking.util.spec.ts @@ -9,10 +9,6 @@ import { ILIKE_RESCUE_BAND_TOP, ILIKE_RESCUE_BAND_BOTTOM, RankableMemory, - relativeRescueScore, - RELATIVE_FTS_MAX_BOOST, - RELATIVE_ILIKE_MAX_BOOST, - RELATIVE_LEXICAL_ONLY_ANCHOR, } from './memory-ranking.util'; describe('memory-ranking.util', () => { @@ -218,98 +214,4 @@ describe('memory-ranking.util', () => { expect(compareByRankKeys(x, x)).toBe(0); }); }); - - // PROTOTYPE (RECALL_RELATIVE_RESCUE=true). Default behaviour is unaffected. - describe('relativeRescueScore', () => { - it('never exceeds the cosine ceiling by more than the boost', () => { - const s = relativeRescueScore(1.0, 1, 0, RELATIVE_FTS_MAX_BOOST, 1.0); - expect(s).toBeLessThanOrEqual(1 + RELATIVE_FTS_MAX_BOOST + 1e-6); - }); - - it('is monotonically increasing in cosine at equal lexical quality', () => { - const weakSemantic = relativeRescueScore( - 0.4, - 1, - 0, - RELATIVE_FTS_MAX_BOOST, - 0.9, - ); - const strongSemantic = relativeRescueScore( - 0.9, - 1, - 5, - RELATIVE_FTS_MAX_BOOST, - 0.9, - ); - // This is the property the band inverts: under the band the rank-0 - // lexical hit wins regardless of cosine. - expect(strongSemantic).toBeGreaterThan(weakSemantic); - }); - - it('lets a strong semantic hit beat a perfect lexical hit on a weak one', () => { - // Best-possible FTS agreement on a 0.60-cosine candidate... - const lexicalWinnerOnWeakDoc = relativeRescueScore( - 0.6, - 1, - 0, - RELATIVE_FTS_MAX_BOOST, - 0.95, - ); - // ...still loses to a plain 0.95-cosine candidate with no lexical hit. - expect(lexicalWinnerOnWeakDoc).toBeLessThan(0.95); - }); - - it('still promotes: a lexical hit outranks the same cosine without one', () => { - const cosine = 0.7; - const boosted = relativeRescueScore( - cosine, - 1, - 0, - RELATIVE_ILIKE_MAX_BOOST, - 0.9, - ); - expect(boosted).toBeGreaterThan(cosine); - expect(boosted).toBeCloseTo(cosine * (1 + RELATIVE_ILIKE_MAX_BOOST), 5); - }); - - it('anchors lexical-only candidates below the best semantic hit', () => { - const bestVector = 0.82; - const lexicalOnly = relativeRescueScore( - null, - 1, - 0, - RELATIVE_FTS_MAX_BOOST, - bestVector, - ); - expect(lexicalOnly).toBeLessThan(bestVector); - expect(lexicalOnly).toBeCloseTo( - bestVector * RELATIVE_LEXICAL_ONLY_ANCHOR, - 5, - ); - }); - - it('breaks exact ties strictly by lexical rank', () => { - const a = relativeRescueScore(0.5, 1, 0, RELATIVE_FTS_MAX_BOOST, 0.5); - const b = relativeRescueScore(0.5, 1, 1, RELATIVE_FTS_MAX_BOOST, 0.5); - expect(a).toBeGreaterThan(b); - }); - - it('scales lexical quality: full coverage beats partial at equal cosine', () => { - const full = relativeRescueScore( - 0.6, - 1, - 0, - RELATIVE_ILIKE_MAX_BOOST, - 0.9, - ); - const partial = relativeRescueScore( - 0.6, - 0.33, - 0, - RELATIVE_ILIKE_MAX_BOOST, - 0.9, - ); - expect(full).toBeGreaterThan(partial); - }); - }); }); diff --git a/src/memory/memory-ranking.util.ts b/src/memory/memory-ranking.util.ts index bae37389..6b93bbe3 100644 --- a/src/memory/memory-ranking.util.ts +++ b/src/memory/memory-ranking.util.ts @@ -122,86 +122,6 @@ export function ilikeRescueScore( ); } -/* ─────────────────────────────────────────────────────────────────────────── - * PROTOTYPE (feature-flagged, OFF by default): relative lexical rescue. - * - * See docs/research/memory-formation-query-transform/05-finding-band-inversion.md. - * - * The band design above places every lexical-rescue hit ABOVE the cosine - * ceiling of 1.0. That makes the priority *categorical*: a weak lexical match - * always outranks a strong semantic match, and the candidate's own cosine is - * discarded as an ordering signal even when it is available. On a corpus whose - * distractors restate the query verbatim, the entire band is distractors. - * - * Relative mode instead treats lexical agreement as a *bounded boost on the - * semantic score* rather than a separate stratum: - * - * score = cosine * (1 + maxBoost * quality) - * - * Consequences: - * - a lexical hit can promote a candidate, but a candidate with a much better - * cosine can still beat it (the boost is bounded by `maxBoost`); - * - the cosine ordering inside the rescued set is preserved instead of being - * overwritten by ts_rank, which is what the band does today; - * - all scores stay in the same numeric scale, so downstream stages - * (usage weighting, the fallback blend, the cross-encoder) compare like - * with like. - * - * Lexical-only candidates (matched by FTS/ILIKE but absent from the vector - * pool) have no cosine to anchor to. They are admitted just below the best - * observed cosine, so they enter the candidate pool but cannot displace the - * best semantic hit. - * ────────────────────────────────────────────────────────────────────────── */ - -/** Maximum multiplicative promotion an FTS/BM25 agreement may buy. */ -export const RELATIVE_FTS_MAX_BOOST = 0.2; -/** Maximum multiplicative promotion an ILIKE coverage agreement may buy. */ -export const RELATIVE_ILIKE_MAX_BOOST = 0.1; -/** Maximum multiplicative promotion the identity-profile rescue may buy. */ -export const RELATIVE_IDENTITY_MAX_BOOST = 0.15; -/** - * A lexical-only candidate (no vector hit) is anchored at this fraction of the - * best observed cosine, so it can never outrank the best semantic hit. - */ -export const RELATIVE_LEXICAL_ONLY_ANCHOR = 0.9; - -/** - * Relative-mode rescue score. - * - * @param cosine the candidate's own cosine similarity, or null/undefined if - * it was not in the vector pool - * @param quality normalised lexical agreement in [0, 1] (ts_rank / max - * ts_rank for FTS, matched/total terms for ILIKE) - * @param rankIndex position in the lexical result set, used only as a strict - * tie-break so two candidates never collide - * @param maxBoost the ceiling on the promotion this signal may buy - * @param bestVector the best cosine observed in this query's vector pool, - * used to anchor lexical-only candidates - */ -export function relativeRescueScore( - cosine: number | null | undefined, - quality: number, - rankIndex: number, - maxBoost: number, - bestVector: number, -): number { - const q = Number.isFinite(quality) ? Math.min(1, Math.max(0, quality)) : 0; - // Positional term is deliberately tiny: it exists only to break exact ties. - const positional = 1e-6 * rrfNorm(rankIndex); - - if (cosine != null && Number.isFinite(cosine) && cosine > 0) { - return cosine * (1 + maxBoost * q) + positional; - } - - // Lexical-only: no semantic evidence at all. Anchor strictly below the best - // semantic hit so it is admitted to the pool but cannot win on lexical - // agreement alone. - const anchor = - (Number.isFinite(bestVector) && bestVector > 0 ? bestVector : 0.5) * - RELATIVE_LEXICAL_ONLY_ANCHOR; - return anchor * (0.5 + 0.5 * q) + positional; -} - /** Minimal shape the comparator needs. Deliberately structural, not `Memory`. */ export interface RankableMemory { id?: string; From 8af07f6f893a777229fda7fbbeefcb3a04e4ffc7 Mon Sep 17 00:00:00 2001 From: heybeaux Date: Sun, 23 Aug 2026 22:45:16 -0700 Subject: [PATCH 8/8] fix(recall): require lexical coverage for rescue --- scripts/research/run-scale-fix-arms.sh | 73 +++++--- src/memory/memory-query-defect-ab.spec.ts | 206 ++++++++++++++++++++++ src/memory/memory-query.service.ts | 83 ++++++++- src/memory/memory-ranking.util.spec.ts | 28 +++ src/memory/memory-ranking.util.ts | 33 ++++ 5 files changed, 394 insertions(+), 29 deletions(-) create mode 100644 src/memory/memory-query-defect-ab.spec.ts diff --git a/scripts/research/run-scale-fix-arms.sh b/scripts/research/run-scale-fix-arms.sh index 48b2d442..d551fea2 100755 --- a/scripts/research/run-scale-fix-arms.sh +++ b/scripts/research/run-scale-fix-arms.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash -# Three-arm retrieval A/B on the noisy mnemon corpus. +# Multi-arm retrieval A/B on the noisy mnemon corpus. # -# default — shipped behaviour (banded rescue + raw-score sticky re-add) -# scalefix — RECALL_RERANK_SCALE_FIX=true (rank full pool, no raw re-add) -# norescue — RECALL_NO_RESCUE=true (vector-only control) +# default — shipped behaviour (banded rescue + raw-score sticky re-add) +# scalefix — RECALL_RERANK_SCALE_FIX=true (rank full pool, no raw re-add) +# norescue — RECALL_NO_RESCUE=true (vector-only control) +# covfloor — scalefix + RECALL_LEXICAL_COVERAGE_FLOOR (defect A) +# tiebreak — scalefix + RECALL_RESCUE_SQL_TIEBREAK (defect B) +# all — scalefix + defects A and B # # Originally five arms: the RECALL_RELATIVE_RESCUE prototype ("relative") and # its combination with the scale fix ("both") have been removed along with the @@ -18,15 +21,26 @@ # usage-weighted reranking feeds them back at weight 0.15 — without the reset # arm N is measured against a system arm N-1 mutated. # -# Usage: scripts/research/run-scale-fix-arms.sh +# The API key must belong to an `agents` row in the probe DB: the LAN-trust +# branch of ApiKeyGuard resolves X-AM-User-ID only when the key hashes to a +# known agent, and silently falls back to the account's *default* user (which +# owns no memories, so every query returns an empty page) when it does not. +# +# Usage: scripts/research/run-scale-fix-arms.sh [arm ...] set -euo pipefail REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" KEY="${1:-${PROBE_API_KEY:-}}" +shift || true DB="${PROBE_DB_URL:-postgresql://engram:engram@localhost:5432/engram_ablation_prefix}" PORT="${PROBE_PORT:-3007}" +# Artifact filenames are "${ARM_OUT_PREFIX}.json". Override the prefix to +# keep a previous run's artifacts around for comparison instead of clobbering +# them — re-running a set over the top of an older one destroys the only +# evidence that the older set was measured against different code. +OUT_PREFIX="${ARM_OUT_PREFIX:-/tmp/arm-}" -[ -n "$KEY" ] || { echo "usage: $0 " >&2; exit 2; } +[ -n "$KEY" ] || { echo "usage: $0 [arm ...]" >&2; exit 2; } reset_usage() { psql "$DB" -q -c " @@ -40,13 +54,36 @@ reset_usage() { WHERE m.id = s.id;" >/dev/null } +# arm : SCALE_FIX : NO_RESCUE : LEXICAL_COVERAGE_FLOOR : RESCUE_SQL_TIEBREAK +ALL_ARMS=( + "default:false:false:false:false" + "scalefix:true:false:false:false" + "norescue:false:true:false:false" + "covfloor:true:false:true:false" + "tiebreak:true:false:false:true" + "all:true:false:true:true" +) + +ARMS=() +if [ $# -eq 0 ]; then + ARMS=("${ALL_ARMS[@]}") +else + for want in "$@"; do + for spec in "${ALL_ARMS[@]}"; do + [ "${spec%%:*}" = "$want" ] && ARMS+=("$spec") + done + done +fi + start_server() { - local arm="$1" scalefix="$2" norescue="$3" + local arm="$1" scalefix="$2" norescue="$3" covfloor="$4" tiebreak="$5" pkill -f "node dist/main.js" 2>/dev/null || true sleep 2 cd "$REPO" RECALL_RERANK_SCALE_FIX="$scalefix" \ RECALL_NO_RESCUE="$norescue" \ + RECALL_LEXICAL_COVERAGE_FLOOR="$covfloor" \ + RECALL_RESCUE_SQL_TIEBREAK="$tiebreak" \ nohup node dist/main.js >"/tmp/engram-arm-${arm}.log" 2>&1 & for _ in $(seq 1 60); do if curl -sf -m 2 "http://127.0.0.1:${PORT}/v1/health" >/dev/null 2>&1; then return 0; fi @@ -55,22 +92,16 @@ start_server() { echo "server did not come up (arm=$arm)" >&2; exit 1 } -# arm : scalefix : norescue -ARMS=( - "default:false:false" - "scalefix:true:false" - "norescue:false:true" -) - +OUTS=() for SPEC in "${ARMS[@]}"; do - IFS=: read -r ARM SF NR <<<"$SPEC" - echo "### arm=$ARM scalefix=$SF norescue=$NR" - start_server "$ARM" "$SF" "$NR" + IFS=: read -r ARM SF NR CF TB <<<"$SPEC" + echo "### arm=$ARM scalefix=$SF norescue=$NR covfloor=$CF tiebreak=$TB" + start_server "$ARM" "$SF" "$NR" "$CF" "$TB" reset_usage PROBE_API_KEY="$KEY" node "$REPO/scripts/research/retrieval-probe.mjs" \ - --limit 10 --label "$ARM" --out "/tmp/arm-${ARM}.json" \ - | tail -12 + --limit 10 --label "$ARM" --out "${OUT_PREFIX}${ARM}.json" \ + | tail -4 + OUTS+=("${OUT_PREFIX}${ARM}.json") done -node "$REPO/scripts/research/score-arms.mjs" \ - /tmp/arm-default.json /tmp/arm-scalefix.json /tmp/arm-norescue.json +node "$REPO/scripts/research/score-arms.mjs" "${OUTS[@]}" diff --git a/src/memory/memory-query-defect-ab.spec.ts b/src/memory/memory-query-defect-ab.spec.ts new file mode 100644 index 00000000..ada614e7 --- /dev/null +++ b/src/memory/memory-query-defect-ab.spec.ts @@ -0,0 +1,206 @@ +/** + * Research: RECALL_LEXICAL_COVERAGE_FLOOR / RECALL_RESCUE_SQL_TIEBREAK. + * + * Two defects, measured on the noisy mnemon corpus and written up in + * docs/research/memory-formation-query-transform/06-finding-rescue-admission.md. + * + * Defect A — one 71-character junk memory held rank 0 on 20/20 benchmark + * queries. NOT `ts_rank` short-document bias, as originally hypothesised: the + * `OR`-joined ILIKE rescue promotes a row matching ONE of eight extracted terms + * into the (1.00, 1.10] band, above every cosine hit. The coverage floor alone + * takes gold@1 from 0/20 to 14/20. + * + * Defect B — the rescue SQL has no tiebreak, so which member of a cluster of + * equally-ranked rows survives `LIMIT 100` / `LIMIT 20` is decided by Postgres' + * physical row order and changes across restarts. Measured on repeat runs of + * the same build: 28 and 104 top-10 slot changes with the flag off, 0 and 0 + * with it on. + * + * Every test pins BOTH states: flag off must reproduce shipped behaviour + * exactly, flag on must fix the defect. + */ +import { MemoryQueryService } from './memory-query.service'; +import { MemoryQueryRankingService } from './memory-query-ranking.service'; +import { MemoryQueryContextService } from './memory-query-context.service'; +import { PrismaService } from '../prisma/prisma.service'; +import { EmbeddingService } from './embedding.service'; +import { TemporalParserService } from './temporal/temporal-parser.service'; +import { RecallWeightService } from './recall-weight.service'; + +/** Eight rescue terms survive extraction (length >= 4, not a stop word). */ +const QUERY = 'benchmark project fact user creation endpoint schema validation'; +const POOL_SIZE = 10; + +/** The junk memory's stand-in: matches exactly one of the eight terms. */ +const JUNK_ID = 'junk'; + +interface Harness { + service: MemoryQueryService; + ranking: MemoryQueryRankingService; + sql: string[]; +} + +function harness(): Harness { + const memories = [ + ...Array.from({ length: POOL_SIZE }, (_, i) => ({ + id: `m${String(i).padStart(3, '0')}`, + raw: `memory ${i} about ${QUERY}`, + userId: 'user-1', + importanceScore: 0.5, + effectiveScore: 0.5, + createdAt: new Date(2026, 0, 1 + i), + extraction: {}, + })), + { + id: JUNK_ID, + raw: 'benchmark is a person known to cmt69v4ei000e82c9zkmqraa4.', + userId: 'user-1', + importanceScore: 0.5, + effectiveScore: 0.5, + createdAt: new Date(2026, 0, 1), + extraction: {}, + }, + ]; + + const sql: string[] = []; + const prisma = { + memory: { + findMany: jest.fn().mockResolvedValue(memories), + updateMany: jest.fn().mockResolvedValue({ count: 0 }), + }, + $queryRawUnsafe: jest.fn().mockImplementation((statement: string) => { + sql.push(statement); + // FTS finds nothing (websearch_to_tsquery is AND-joined, and the junk row + // shares only one term), so the ILIKE pass is the only rescue in play. + if (statement.includes('ts_rank')) return Promise.resolve([]); + return Promise.resolve([{ id: JUNK_ID, match_count: 1 }]); + }), + } as any as PrismaService; + + const embedding = { + generate: jest.fn().mockResolvedValue([0.1, 0.2, 0.3]), + generateForRecall: jest.fn().mockResolvedValue([0.1, 0.2, 0.3]), + // Junk is the WORST semantic match in the pool. Anything that puts it on + // top did so lexically. + search: jest.fn().mockImplementation(() => + Promise.resolve([ + ...Array.from({ length: POOL_SIZE }, (_, i) => ({ + id: `m${String(i).padStart(3, '0')}`, + score: 0.9 - i * 0.01, + })), + { id: JUNK_ID, score: 0.1 }, + ]), + ), + } as any as EmbeddingService; + + const temporalParser = { + parse: jest + .fn() + .mockReturnValue({ semanticQuery: QUERY, temporalFilter: null }), + } as any as TemporalParserService; + + const recallWeightService = { + recallWeight: jest.fn().mockReturnValue(1.0), + applyUsageWeighting: jest + .fn() + .mockImplementation((mems: any[]) => Promise.resolve(mems)), + } as any as RecallWeightService; + + const ranking = new MemoryQueryRankingService( + prisma, + embedding, + recallWeightService, + ); + jest + .spyOn(ranking, 'surfaceInsights') + .mockImplementation((existing: any) => Promise.resolve(existing)); + + const service = new MemoryQueryService( + prisma, + embedding, + temporalParser, + recallWeightService, + ranking, + new MemoryQueryContextService(prisma), + ); + + return { service, ranking, sql }; +} + +async function pageIds(service: MemoryQueryService): Promise { + const res = await service.recall('user-1', { + query: QUERY, + limit: 5, + } as any); + return res.memories.map((m) => m.id); +} + +describe('recall ranking: defect A + B research flags', () => { + const savedEnv = { ...process.env }; + + afterEach(() => { + process.env = { ...savedEnv }; + jest.restoreAllMocks(); + }); + + describe('RECALL_LEXICAL_COVERAGE_FLOOR (defect A, part 1)', () => { + it('flag OFF: one matched term of eight still wins rank 0 (the defect)', async () => { + delete process.env.RECALL_LEXICAL_COVERAGE_FLOOR; + const { service } = harness(); + const ids = await pageIds(service); + expect(ids[0]).toBe(JUNK_ID); + }); + + it('flag ON: the one-term match is not promoted above the cosine ceiling', async () => { + process.env.RECALL_LEXICAL_COVERAGE_FLOOR = 'true'; + const { service } = harness(); + const ids = await pageIds(service); + expect(ids[0]).toBe('m000'); + expect(ids).not.toContain(JUNK_ID); + }); + + it('flag ON: the row is demoted, not deleted — it stays in the deep pool', async () => { + process.env.RECALL_LEXICAL_COVERAGE_FLOOR = 'true'; + const { service } = harness(); + const res = await service.recall('user-1', { + query: QUERY, + limit: 50, + } as any); + expect(res.memories.map((m) => m.id)).toContain(JUNK_ID); + }); + }); + + describe('RECALL_RESCUE_SQL_TIEBREAK (defect B)', () => { + it('flag OFF: rescue SQL emits no id tiebreak (shipped statements)', async () => { + delete process.env.RECALL_RESCUE_SQL_TIEBREAK; + const { service, sql } = harness(); + await pageIds(service); + expect(sql.length).toBeGreaterThan(0); + for (const statement of sql) { + expect(statement).not.toMatch(/id ASC/); + } + }); + + it('flag ON: every rescue ORDER BY ends with a unique-key tiebreak', async () => { + process.env.RECALL_RESCUE_SQL_TIEBREAK = 'true'; + const { service, sql } = harness(); + await pageIds(service); + const ordered = sql.filter((s) => s.includes('ORDER BY')); + expect(ordered.length).toBeGreaterThan(0); + for (const statement of ordered) { + expect(statement).toMatch(/ORDER BY[^;]*?id ASC\s*\n?\s*LIMIT/); + } + }); + }); + + describe('all flags off', () => { + it('leaves the shipped ordering untouched', async () => { + delete process.env.RECALL_LEXICAL_COVERAGE_FLOOR; + delete process.env.RECALL_RESCUE_SQL_TIEBREAK; + const { service, sql } = harness(); + const ids = await pageIds(service); + expect(ids[0]).toBe(JUNK_ID); + expect(sql.join('\n')).not.toMatch(/id ASC/); + }); + }); +}); diff --git a/src/memory/memory-query.service.ts b/src/memory/memory-query.service.ts index 6130c440..8293480b 100644 --- a/src/memory/memory-query.service.ts +++ b/src/memory/memory-query.service.ts @@ -46,6 +46,7 @@ import { compareByRankKeys, ftsRescueScore, ilikeRescueScore, + meetsLexicalCoverageFloor, ILIKE_RESCUE_BAND_BOTTOM, } from './memory-ranking.util'; @@ -87,6 +88,47 @@ export class MemoryQueryService { */ private readonly noRescue = process.env.RECALL_NO_RESCUE === 'true'; + /** + * RESEARCH FLAG — default OFF. Defect A, part 1. + * + * `RECALL_LEXICAL_COVERAGE_FLOOR=true` stops the `OR`-joined ILIKE rescue + * from promoting a row that contains a single incidental query token into the + * (1.00, 1.10] band, where it outranks every cosine hit including perfect + * semantic matches. On the noisy corpus one 71-character junk memory matched + * exactly one of the eight extracted terms ("mnemon", which appears in all 20 + * task queries) and was promoted on 20/20 queries for it. + * + * Rows below the floor are NOT removed from the candidate pool — they simply + * keep their cosine score instead of being lifted above the cosine ceiling. + */ + private readonly lexicalCoverageFloor = + process.env.RECALL_LEXICAL_COVERAGE_FLOOR === 'true'; + + /** + * RESEARCH FLAG — default OFF. Defect B. + * + * `RECALL_RESCUE_SQL_TIEBREAK=true` appends `id ASC` to every rescue query's + * `ORDER BY`. Without it, `ORDER BY ts_rank DESC` / `ORDER BY importance_score + * DESC, created_at DESC` leave ties to Postgres' physical row order, which is + * not stable across server restarts, VACUUMs or plan changes. On the + * benchmark corpus the gold memory sits in a cluster of ~11 near-identical + * rows with equal `ts_rank`, so *which* member the `LIMIT 100`/`LIMIT 20` + * window keeps — and hence which one gets the rank-0 RRF term — is decided by + * heap order. `compareByRankKeys` only imposes a total order on rows that + * already came back; this imposes one on the rows that come back at all. + */ + private readonly rescueSqlTiebreak = + process.env.RECALL_RESCUE_SQL_TIEBREAK === 'true'; + + /** + * `, ASC` fragment appended to rescue `ORDER BY` clauses, or the empty + * string when {@link rescueSqlTiebreak} is off — so flag-off SQL is + * character-identical to the shipped statements. + */ + private rescueTiebreak(column = 'id'): string { + return this.rescueSqlTiebreak ? `, ${column} ASC` : ''; + } + constructor( private prisma: PrismaService, private embedding: EmbeddingService, @@ -453,7 +495,7 @@ export class MemoryQueryService { AND m.searchable IS NOT FALSE AND m.embedding_status != 'DUPLICATE' AND m.is_duplicate_of IS NULL - ORDER BY ts_rank(to_tsvector('english', m.raw), websearch_to_tsquery('english', $2)) DESC + ORDER BY ts_rank(to_tsvector('english', m.raw), websearch_to_tsquery('english', $2)) DESC${this.rescueTiebreak('m.id')} LIMIT 100`, poolIds, searchQuery, @@ -470,7 +512,7 @@ export class MemoryQueryService { AND searchable IS NOT FALSE AND embedding_status != 'DUPLICATE' AND is_duplicate_of IS NULL - ORDER BY ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $2)) DESC + ORDER BY ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $2)) DESC${this.rescueTiebreak()} LIMIT 100`, resolvedUserIds, searchQuery, @@ -485,7 +527,7 @@ export class MemoryQueryService { AND searchable IS NOT FALSE AND embedding_status != 'DUPLICATE' AND is_duplicate_of IS NULL - ORDER BY ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $1)) DESC + ORDER BY ts_rank(to_tsvector('english', raw), websearch_to_tsquery('english', $1)) DESC${this.rescueTiebreak()} LIMIT 100`, searchQuery, ); @@ -567,7 +609,7 @@ export class MemoryQueryService { AND m.searchable IS NOT FALSE AND m.embedding_status != 'DUPLICATE' AND m.is_duplicate_of IS NULL - ORDER BY m.importance_score DESC, m.created_at DESC + ORDER BY m.importance_score DESC, m.created_at DESC${this.rescueTiebreak('m.id')} LIMIT 20`, poolIds, ...ilikeParams, @@ -583,7 +625,7 @@ export class MemoryQueryService { AND searchable IS NOT FALSE AND embedding_status != 'DUPLICATE' AND is_duplicate_of IS NULL - ORDER BY importance_score DESC, created_at DESC + ORDER BY importance_score DESC, created_at DESC${this.rescueTiebreak()} LIMIT 20`, resolvedUserIds, ...ilikeParams, @@ -597,7 +639,7 @@ export class MemoryQueryService { AND searchable IS NOT FALSE AND embedding_status != 'DUPLICATE' AND is_duplicate_of IS NULL - ORDER BY importance_score DESC, created_at DESC + ORDER BY importance_score DESC, created_at DESC${this.rescueTiebreak()} LIMIT 20`, ...ilikeParams, ); @@ -610,16 +652,37 @@ export class MemoryQueryService { // above the cosine ceiling of 1.0, never reaching the FTS band — // and are strictly decreasing with rank at equal coverage. let ilikeAdded = 0; + let ilikeBandRank = 0; for ( let ilikeRank = 0; ilikeRank < ilikeResults.length; ilikeRank++ ) { const row = ilikeResults[ilikeRank]; + // Defect A, part 1 (flag-gated): a row containing exactly one of + // the extracted terms is a coincidence, not a lexical agreement, + // and must not be lifted above the cosine ceiling. Skipping it + // here leaves any cosine score it already has untouched and keeps + // it out of `keywordRescueIds`, so it also loses the sticky + // re-add protection it never earned. + if ( + this.lexicalCoverageFloor && + !meetsLexicalCoverageFloor( + Number(row.match_count), + words.length, + ) + ) { + continue; + } + // Position within the *promoted* set, so skipping a row does not + // leave a hole in the RRF positional term. Identical to + // `ilikeRank` whenever the floor is off, because nothing is + // skipped then. + const bandRank = ilikeBandRank++; const bandedScore = ilikeRescueScore( Number(row.match_count), words.length, - ilikeRank, + bandRank, ); ftsResultIds.add(row.id); keywordRescueIds.add(row.id); @@ -674,7 +737,11 @@ export class MemoryQueryService { ...sessionIdFilter, ...(dto.filterAgentId ? { agentId: dto.filterAgentId } : {}), }, - orderBy: [{ importanceScore: 'desc' }, { createdAt: 'desc' }], + orderBy: [ + { importanceScore: 'desc' }, + { createdAt: 'desc' }, + ...(this.rescueSqlTiebreak ? [{ id: 'asc' } as const] : []), + ], take: 10, select: { id: true }, }); diff --git a/src/memory/memory-ranking.util.spec.ts b/src/memory/memory-ranking.util.spec.ts index 6a7c4ff6..097a1b3d 100644 --- a/src/memory/memory-ranking.util.spec.ts +++ b/src/memory/memory-ranking.util.spec.ts @@ -8,10 +8,38 @@ import { FTS_RESCUE_BAND_BOTTOM, ILIKE_RESCUE_BAND_TOP, ILIKE_RESCUE_BAND_BOTTOM, + meetsLexicalCoverageFloor, RankableMemory, } from './memory-ranking.util'; describe('memory-ranking.util', () => { + describe('meetsLexicalCoverageFloor (defect A, part 1)', () => { + it('rejects a single incidental term out of many', () => { + // The exact benchmark case: an eight-term query, and a 71-character junk + // memory whose only overlap is the corpus-wide token "mnemon". + expect(meetsLexicalCoverageFloor(1, 8)).toBe(false); + }); + + it('accepts two or more matched terms', () => { + expect(meetsLexicalCoverageFloor(2, 8)).toBe(true); + expect(meetsLexicalCoverageFloor(8, 8)).toBe(true); + }); + + it('degrades to "match the only term" for single-term queries', () => { + expect(meetsLexicalCoverageFloor(1, 1)).toBe(true); + expect(meetsLexicalCoverageFloor(0, 1)).toBe(false); + }); + + it('treats a garbled match count as not clearing the floor', () => { + expect(meetsLexicalCoverageFloor(Number.NaN, 8)).toBe(false); + }); + + it('never rejects on a zero-term query (no terms, no floor to clear)', () => { + // totalTerms 0 is coerced to 1, so the rule is still "match what exists". + expect(meetsLexicalCoverageFloor(1, 0)).toBe(true); + }); + }); + describe('rrfNorm', () => { it('is 1 at rank 0 and strictly decreasing', () => { expect(rrfNorm(0)).toBe(1); diff --git a/src/memory/memory-ranking.util.ts b/src/memory/memory-ranking.util.ts index 6b93bbe3..080e9c49 100644 --- a/src/memory/memory-ranking.util.ts +++ b/src/memory/memory-ranking.util.ts @@ -122,6 +122,39 @@ export function ilikeRescueScore( ); } +/** + * Minimum number of extracted rescue terms a row must actually contain before + * the ILIKE path is allowed to promote it ABOVE the cosine ceiling. + * + * The ILIKE rescue SQL is `OR`-joined, so a row matching a single incidental + * token out of eight enters the (1.00, 1.10] band and therefore outranks a + * perfect semantic match. That is not a lexical agreement, it is a coincidence. + * The floor is structural rather than tuned: "one token is not a match, two + * are", degrading to "match the only term there is" for single-term queries. + * + * See docs/research/memory-formation-query-transform/06-finding-rescue-admission.md. + */ +export const MIN_LEXICAL_RESCUE_TERMS = 2; + +/** + * Does this ILIKE row clear the coverage floor? + * + * `matched` is the SQL-computed count of rescue terms present in the row and + * `total` the number of terms extracted from the query. A row below the floor + * is not *dropped* — it simply is not promoted into the rescue band, so it + * keeps whatever cosine score the vector search gave it (or none at all, if the + * vector search never surfaced it). + */ +export function meetsLexicalCoverageFloor( + matched: number, + total: number, +): boolean { + const totalTerms = total > 0 ? total : 1; + const required = Math.min(MIN_LEXICAL_RESCUE_TERMS, totalTerms); + const safeMatched = Number.isFinite(matched) ? Math.floor(matched) : 0; + return safeMatched >= required; +} + /** Minimal shape the comparator needs. Deliberately structural, not `Memory`. */ export interface RankableMemory { id?: string;