fix: preserve memory integrity and correct ensemble recall

.github/workflows/benchmark.yml

@@ -136,16 +136,18 @@ jobs:
             }
 
             // Extract the report section
-            const reportStart = output.indexOf('╔══');
-            const reportEnd = output.lastIndexOf('benchmark PASSED') !== -1
-              ? output.lastIndexOf('benchmark PASSED') + 20
-              : output.lastIndexOf('benchmark FAILED') + 20;
+            const reportStart = output.indexOf('Engram recall benchmark (metric version ');
+            const successMarker = 'Quality thresholds passed';
+            const failureMarker = 'Quality thresholds FAILED';
+            const reportEnd = output.lastIndexOf(successMarker) !== -1
+              ? output.lastIndexOf(successMarker) + successMarker.length
+              : output.lastIndexOf(failureMarker) + failureMarker.length;
 
             const report = reportStart >= 0
               ? output.slice(reportStart, reportEnd > reportStart ? reportEnd : undefined)
               : 'Report not found in output';
 
-            const passed = output.includes('ALL THRESHOLDS PASSED') && !output.includes('THRESHOLDS NOT MET') && !output.includes('Benchmark FAILED');
+            const passed = output.includes('Quality thresholds passed') && !output.includes('Quality thresholds FAILED') && !output.includes('Benchmark FAILED');
             const emoji = passed ? '✅' : '❌';
 
             const body = `## ${emoji} Recall Benchmark Results
@@ -199,11 +201,11 @@ jobs:
             echo "::error::Benchmark output file not found — benchmark step likely failed"
             exit 1
           fi
-          if ! grep -q "ALL THRESHOLDS PASSED" benchmark-output.txt; then
-            echo "::error::Benchmark did not report ALL THRESHOLDS PASSED"
+          if ! grep -q "Quality thresholds passed" benchmark-output.txt; then
+            echo "::error::Benchmark did not report Quality thresholds passed"
             exit 1
           fi
-          if grep -Eq "THRESHOLDS NOT MET|Benchmark FAILED" benchmark-output.txt; then
+          if grep -Eq "Quality thresholds FAILED|Benchmark FAILED" benchmark-output.txt; then
             echo "::error::Benchmark thresholds not met"
             exit 1
           fi

.github/workflows/ci-cloud.yml

@@ -93,7 +93,7 @@ jobs:
 
   # ── 4a. Test shard — memory (large module, needs own slot) ───────────────
   test-memory:
-    name: Test / memory
+    name: Test / shard 1 of 5
     runs-on: ubuntu-latest
     timeout-minutes: 25
     needs: install
@@ -133,11 +133,13 @@ jobs:
       - name: Test — memory module
         run: |
           pnpm test -- --no-coverage --forceExit --ci \
-            --testPathPatterns='src/memory'
+            --shard=1/5
+      - name: Verify benchmark scoring definitions
+        run: pnpm exec jest --config test/jest-e2e.json --testRegex='benchmark/scoring.spec.ts$' --runInBand --ci
 
   # ── 4b. Test shard — core (identity, account, common) ────────────────────
   test-core:
-    name: Test / core
+    name: Test / shard 2 of 5
     runs-on: ubuntu-latest
     timeout-minutes: 25
     needs: install
@@ -177,11 +179,11 @@ jobs:
       - name: Test — core modules
         run: |
           pnpm test -- --no-coverage --forceExit --ci \
-            --testPathPatterns='src/(identity|account|common|app\.controller)'
+            --shard=2/5
 
   # ── 4b. Test shard — intelligence ────────────────────────────────────────
   test-intelligence:
-    name: Test / intelligence
+    name: Test / shard 3 of 5
     runs-on: ubuntu-latest
     timeout-minutes: 12
     needs: install
@@ -221,11 +223,11 @@ jobs:
       - name: Test — intelligence modules
         run: |
           pnpm test -- --no-coverage --forceExit --ci \
-            --testPathPatterns='src/(deduplication|awareness|multi-query|consolidation|prefetch|retrieval-signals|graph|clustering|hierarchy|ensemble)'
+            --shard=3/5
 
   # ── 4c. Test shard — cloud-edition features ───────────────────────────────
   test-cloud:
-    name: Test / cloud features
+    name: Test / shard 4 of 5 features
     runs-on: ubuntu-latest
     timeout-minutes: 12
     needs: install
@@ -265,11 +267,11 @@ jobs:
       - name: Test — cloud-edition modules
         run: |
           pnpm test -- --no-coverage --forceExit --ci \
-            --testPathPatterns='src/(cloud-link|cloud-sync|instance|webhooks|analytics|monitoring|stripe|reembedding)'
+            --shard=4/5
 
   # ── 4d. Test shard — remaining features ──────────────────────────────────
   test-features:
-    name: Test / features
+    name: Test / shard 5 of 5
     runs-on: ubuntu-latest
     timeout-minutes: 12
     needs: install
@@ -309,7 +311,7 @@ jobs:
       - name: Test — remaining feature modules
         run: |
           pnpm test -- --no-coverage --forceExit --ci \
-            --testPathPatterns='src/(llm|import|import-v2|inbound-email|agent|agent-recall|agent-session|anticipatory|auto|correction|entity-profile|events|feedback|fog-index|health|memory-access-log|memory-pool|queue|scoped-context|scripts|session-indexing|storage|summarization|teams|utils|eval|dashboard)'
+            --shard=5/5
 
   # ── 5. Gate ───────────────────────────────────────────────────────────────
   tests-passed:

README.md

@@ -227,46 +227,11 @@ pnpm longmemeval --subset full --resume results/full-<ts>.jsonl       # Resume a
 
 ## Recall Benchmark
 
-Engram includes a comprehensive recall benchmark suite that tests semantic retrieval quality across 81 queries in 7 categories. Every PR runs the benchmark in CI with real embeddings (bge-base-en-v1.5) and ensemble reranking.
-
-### Latest Results (March 2026)
-
-| Metric | Pre-Dream Cycle | Post-Dream Cycle |
-|--------|:-:|:-:|
-| **Precision@5** | 95.1% ✅ | 96.9% ✅ |
-| **Recall@20** | 95.7% | 96.9% |
-| **MRR** | 0.836 | 0.874 |
-| **Isolation** | 100% ✅ | 100% ✅ |
-| **Queries Passed** | 78/81 | 79/81 |
-
-<details>
-<summary>Category Breakdown (Post-Dream Cycle)</summary>
-
-| Category | Queries | Passed | P@5 | R@20 | MRR | Isolation |
-|----------|:-------:|:------:|:---:|:----:|:---:|:---------:|
-| Adversarial | 10 | 10 | 100% | 100% | 1.00 | 100% |
-| Cross-feature | 10 | 10 | 100% | 100% | 0.88 | 100% |
-| Edge case | 16 | 16 | 100% | 100% | 0.95 | 100% |
-| Emotional | 10 | 9 | 85% | 95% | 0.73 | 100% |
-| RLS isolation | 10 | 10 | 100% | 100% | 0.88 | 100% |
-| Semantic | 14 | 13 | 93% | 100% | 0.79 | 100% |
-| Temporal | 11 | 11 | 100% | 82% | 0.87 | 100% |
-
-</details>
-
-### Benchmark Progression
-
-The recall pipeline evolved through several iterations:
-
-| Phase | P@5 | Notes |
-|-------|:---:|-------|
-| Baseline (cosine only) | ~45% | Single-model vector search, no reranking |
-| + BM25 hybrid | ~65% | Full-text search fusion for keyword recall |
-| + Sentiment polarity | ~75% | Penalizes opposite-emotion memories |
-| + Ensemble reranking | ~85% | 2 cross-encoder models via RRF |
-| + Fixture enrichment | ~90% | Realistic gold memories, noise calibration |
-| + Temporal patterns | 95.1% | Month/year/week recognition, expanded BM25 pool |
-| + Dream Cycle | **96.9%** | Post-consolidation: cleaner corpus, higher signal |
+The recall suite exercises 81 fixture queries in seven categories. Metric version 2 reports **required-item coverage@5** on the 57 queries with top-five labels, **expected-item coverage@20** on the ten queries with top-20 labels, and reciprocal rank of the first required hit. Missing labels are excluded, not scored as perfect. An empty expected list is not an explicit no-answer judgment.
+
+The previous published 95.1%/96.9% “precision” and associated recall/MRR tables used a different, inflated scoring definition. Those historical numbers are withdrawn as precision claims and must not be compared directly with version 2. Partial required-item labels do not establish conventional precision or exhaustive recall; fully judged held-out queries are still needed.
+
+The real-embedding suite always enforces the unchanged 95% required-coverage gate, forbidden-ID isolation, and any explicitly labelled no-answer cases. Reports include label denominators and a metric version. A passing forbidden-ID fixture does not prove production database-role isolation.
 
 ### Running the Benchmark
 

scripts/audit/consolidation-reconciliation.sql

@@ -0,0 +1,26 @@
+-- READ ONLY inventory. Supply the deployment's actual retrieval model and dims:
+-- psql ... -v model_id=bge-base -v expected_dims=768 -f scripts/audit/consolidation-reconciliation.sql
+-- No repairs are performed. Snapshot/back up before planning any restoration.
+BEGIN READ ONLY;
+WITH replacements AS (
+  SELECT s.user_id, s.consolidated_into AS replacement_id,
+         count(*) AS linked_sources,
+         count(*) FILTER (WHERE s.deleted_at IS NULL AND s.superseded_by_id IS NULL AND s.searchable) AS active_sources,
+         array_agg(s.id ORDER BY s.id) AS source_ids
+  FROM memories s
+  WHERE s.consolidated_into IS NOT NULL
+  GROUP BY s.user_id, s.consolidated_into
+)
+SELECT r.*, m.id IS NULL AS missing_replacement,
+       m.user_id IS DISTINCT FROM r.user_id AS ownership_mismatch,
+       m.searchable, m.deleted_at IS NOT NULL AS replacement_deleted,
+       me.model_id, me.dimensions, vector_dims(me.embedding) AS actual_dimensions
+FROM replacements r
+LEFT JOIN memories m ON m.id = r.replacement_id
+LEFT JOIN memory_embeddings me ON me.memory_id = m.id AND me.model_id = :'model_id'
+WHERE m.id IS NULL OR m.user_id IS DISTINCT FROM r.user_id
+   OR m.deleted_at IS NOT NULL OR NOT m.searchable OR m.superseded_by_id IS NOT NULL
+   OR me.embedding IS NULL OR me.dimensions <> :expected_dims
+   OR vector_dims(me.embedding) <> :expected_dims
+ORDER BY r.user_id, r.replacement_id;
+ROLLBACK;

scripts/audit/preservation-postgres.cjs

@@ -0,0 +1,97 @@
+// Isolated H1 regression: never reads application DATABASE_URL or touches its DB.
+// Run: node scripts/audit/preservation-postgres.cjs (local socket /tmp, pgvector).
+const path = require('node:path');
+const assert = require('node:assert/strict');
+const repo = path.resolve(__dirname, '../..');
+require('ts-node').register({ transpileOnly: true, project: path.join(repo, 'tsconfig.json') });
+const { Client } = require('pg');
+const { DreamCycleConsolidationStage } = require(path.join(repo, 'src/consolidation/stages/dream-cycle-consolidation.stage.ts'));
+const { EmbeddingWriteService } = require(path.join(repo, 'src/vector/embedding-write.service.ts'));
+const { PgVectorEnsembleProvider } = require(path.join(repo, 'src/ensemble/pgvector-ensemble.provider.ts'));
+const db = `engram_preservation_test_${process.pid}`;
+const config = { host: '/tmp', database: 'postgres' };
+(async () => {
+  const admin = new Client(config); await admin.connect();
+  await admin.query(`CREATE DATABASE ${db}`);
+  const tx = new Client({ ...config, database: db });
+  try {
+    await tx.connect();
+    await tx.query('CREATE EXTENSION vector');
+    await tx.query(`CREATE TABLE memories(id text PRIMARY KEY, user_id text DEFAULT 'u', raw text,
+      embedding vector(768), searchable boolean, superseded_by_id text, tier text,
+      consolidated boolean DEFAULT false, consolidated_into text, user_pinned boolean DEFAULT false,
+      deleted_at timestamp, created_at timestamp DEFAULT now())`);
+    await tx.query(`CREATE TABLE memory_embeddings(id text PRIMARY KEY, memory_id text REFERENCES memories(id),
+      model_id text, dimensions int, embedding vector, created_at timestamp, updated_at timestamp,
+      UNIQUE(memory_id, model_id))`);
+    const sql = {
+      $queryRawUnsafe: async (q, ...args) => (await tx.query(q, args)).rows,
+      $executeRawUnsafe: async (q, ...args) => (await tx.query(q, args)).rowCount,
+    };
+    let failPublication = false;
+    const client = { ...sql, memory: {
+      create: async ({ data }) => {
+        await tx.query('INSERT INTO memories(id,raw,searchable,tier) VALUES ($1,$2,$3,$4)', ['new', data.raw, data.searchable, data.tier]);
+        return { id: 'new' };
+      },
+      update: async ({ data }) => tx.query("UPDATE memories SET searchable=$1 WHERE id='new'", [data.searchable]),
+      updateMany: async ({ data }) => {
+        if (failPublication) await tx.query('SELECT 1/0');
+        return tx.query(`UPDATE memories SET consolidated=$1,consolidated_into=$2,
+        superseded_by_id=$3,tier=COALESCE($4,tier) WHERE id IN ('a','b','c')`,
+        [data.consolidated, data.consolidatedInto, data.supersededById ?? null, data.tier ?? null]);
+      },
+    } };
+    const prisma = { $queryRaw: async (strings, ...args) => (await tx.query(strings.reduce((s, part, i) => s + (i ? '$' + i : '') + part, ''), args)).rows, $transaction: async (fn) => {
+      await tx.query('BEGIN');
+      try { const result = await fn(client); await tx.query('COMMIT'); return result; }
+      catch (e) { await tx.query('ROLLBACK'); throw e; }
+    } };
+    const embeddingWriter = new EmbeddingWriteService({
+      $queryRawUnsafe: () => { throw new Error('Used external connection'); },
+      $executeRawUnsafe: () => { throw new Error('Used external connection'); },
+    });
+    for (const [name, model, dims, vector, fail] of [
+      ['768 success', 'bge-base', 768, Array(768).fill(0.1), false],
+      ['1536 success', 'openai-small', 1536, Array(1536).fill(0.1), false],
+      ['publication DB failure', 'bge-base', 768, Array(768).fill(0.1), true],
+      ['missing', 'bge-base', 768, undefined, true],
+      ['sparse', 'bge-base', 768, Array(768), true],
+      ['wrong dimensions', 'bge-base', 768, Array(1536).fill(0.1), true],
+    ]) {
+      failPublication = name === 'publication DB failure';
+      process.env.EMBEDDING_MODEL = model;
+      await tx.query('TRUNCATE memories, memory_embeddings');
+      await tx.query("INSERT INTO memories(id,raw,searchable,tier) VALUES ('a','date: September 12',true,'COLD'),('b','exception: weekends',true,'COLD'),('c','name: Ada',true,'COLD')");
+      const vec = `[${Array(dims).fill(0.1).join(',')}]`;
+      await tx.query('INSERT INTO memory_embeddings SELECT id,id,$1,$2,$3::vector,now(),now() FROM memories', [model, dims, vec]);
+      const stage = new DreamCycleConsolidationStage(prisma, { get: () => undefined },
+        { embed: async () => [vector] }, { chat: async () => ({ content: 'Lossy summary' }) }, embeddingWriter);
+      assert.equal((await stage.fetchColdMemories('u')).length, 3);
+      const call = () => stage.consolidateCluster([{ id: 'a', content: 'date: September 12' }, { id: 'b', content: 'exception: weekends' }, { id: 'c', content: 'name: Ada' }], 'u');
+      if (fail) await assert.rejects(call); else await call();
+      const eligible = (await tx.query(`SELECT m.id, me.dimensions FROM memories m JOIN memory_embeddings me ON me.memory_id=m.id
+        WHERE m.searchable AND m.superseded_by_id IS NULL AND me.model_id=$1 ORDER BY m.id`, [model])).rows;
+      assert.equal(eligible.length, fail ? 3 : 4);
+      assert.deepEqual(eligible.filter(r => r.id !== 'new').map(r => r.id), ['a','b','c']);
+      assert.ok(eligible.every(r => r.dimensions === dims));
+      // Exercise the integrated ensemble reader against the preservation writer's
+      // committed model rows, including rollback paths and a different owner.
+      const provider = new PgVectorEnsembleProvider(prisma);
+      const search = { modelId: model, embedding: Array(dims).fill(0.1), userId: 'u', limit: 10 };
+      const retrieved = await provider.queryByModel(search);
+      assert.deepEqual(retrieved.map(r => r.memoryId).sort(), eligible.map(r => r.id).sort());
+      assert.deepEqual(await provider.queryByModel({ ...search, userId: 'other-owner' }), []);
+      console.log(JSON.stringify({ name, eligible: eligible.length, sourcesPreserved: 3, rollback: fail, ensembleRetrieved: retrieved.length, otherOwnerRetrieved: 0 }));
+    }
+    await tx.query("UPDATE memories SET consolidated_into='missing-replacement' WHERE id IN ('a','b','c')");
+    const inventory = require('node:fs').readFileSync(path.join(__dirname, 'consolidation-reconciliation.sql'), 'utf8')
+      .replace(/:'model_id'/g, "'bge-base'").replace(/:expected_dims/g, '768');
+    const inventoryResult = await tx.query(inventory);
+    assert.equal(inventoryResult[1].rows.length, 1);
+    assert.equal(inventoryResult[1].rows[0].missing_replacement, true);
+    console.log(JSON.stringify({ reconciliationReadOnly: true, missingReplacementDetected: true }));
+  } finally {
+    await tx.end(); await admin.query(`DROP DATABASE ${db}`); await admin.end();
+  }
+})().catch(e => { console.error(e); process.exitCode = 1; });

src/consolidation/stages/dream-cycle-archival.stage.spec.ts

@@ -13,7 +13,11 @@ describe('DreamCycleArchivalStage', () => {
     prisma = {
       memory: {
         findMany: jest.fn().mockResolvedValue([]),
-        updateMany: jest.fn().mockResolvedValue({ count: 0 }),
+        updateMany: jest
+          .fn()
+          .mockImplementation(({ where }) =>
+            Promise.resolve({ count: where.id.in.length }),
+          ),
       },
     };
 
@@ -59,7 +63,12 @@ describe('DreamCycleArchivalStage', () => {
     expect(result.byLayer).toEqual({ SESSION: 1 });
     expect(result.byType).toEqual({ EVENT: 1 });
     expect(prisma.memory.updateMany).toHaveBeenCalledWith({
-      where: { id: { in: ['m1'] }, userId: 'user1' },
+      where: expect.objectContaining({
+        id: { in: ['m1'] },
+        userId: 'user1',
+        userPinned: false,
+        createdAt: { lt: expect.any(Date) },
+      }),
       data: {
         archivedReason: 'low_importance',
         searchable: false,
@@ -272,7 +281,12 @@ describe('DreamCycleArchivalStage', () => {
     expect(result.skippedRecentlyRetrieved).toBe(1);
     expect(result.skippedFrequentlyUsed).toBe(1);
     expect(prisma.memory.updateMany).toHaveBeenCalledWith({
-      where: { id: { in: ['m1'] }, userId: 'user1' },
+      where: expect.objectContaining({
+        id: { in: ['m1'] },
+        userId: 'user1',
+        userPinned: false,
+        createdAt: { lt: expect.any(Date) },
+      }),
       data: expect.objectContaining({ archivedReason: 'low_importance' }),
     });
   });