From 63ecf73c6267092326989510626700225a74e55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=8D=95?= Date: Fri, 5 Jun 2026 09:39:40 +0800 Subject: [PATCH 1/2] feat: implement background memory maintenance for embedding operations and enhance save functionality --- docs/api/mcp.md | 2 + docs/getting-started/lifecycle.md | 2 + src/database/services/save-memory.ts | 36 ++++- src/modules/memory/background-maintenance.ts | 72 ++++++++++ src/modules/memory/save-memory.ts | 39 +++-- src/modules/project/context.ts | 26 ++++ .../memory/save-memory-embedding.test.ts | 133 +++++++++++++++++- .../protocol/retrieval-evals.test.ts | 2 + .../providers/protocol/warm-up.test.ts | 1 + 9 files changed, 293 insertions(+), 20 deletions(-) create mode 100644 src/modules/memory/background-maintenance.ts diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 1fdb707f..71cf97d7 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -39,6 +39,8 @@ Tools are lower-level callable operations used by agents and debugging workflows MCP tools validate project health silently before doing work. If memory is not initialized or a derived-memory rebuild is required, the tool fails with a short actionable error instead of returning status context. +Save tools return after durable memory is written. Changed-project refresh, embedding generation, and sqlite-vec indexing continue as best-effort background maintenance, so semantic retrieval can lag briefly after a save. Run `konteks rebuild` when you need an explicit full repair pass. + Example durable memory payload: ```json diff --git a/docs/getting-started/lifecycle.md b/docs/getting-started/lifecycle.md index e8d975ef..2e5598bf 100644 --- a/docs/getting-started/lifecycle.md +++ b/docs/getting-started/lifecycle.md @@ -66,6 +66,8 @@ Use the `/konteks-save` prompt to persist the outcome of the current agent sessi A single session can contain one task or several related tasks. The agent saves compact structured durable memories first, then writes one session diary summarizing the outcome. Durable memories should be future-useful rules, decisions, constraints, preferences, blockers, or code insights. The diary should be a compact summary of the task, outcome, verification, unresolved questions, and exact next steps. +Save confirms durable persistence immediately. Project refresh and vector indexing may finish shortly afterward in background maintenance. + > [!TIP] > Recommendation: Prefer saving when the session is complete or about to be closed. If progress is partial, the session diary should include pending items and exact next steps. diff --git a/src/database/services/save-memory.ts b/src/database/services/save-memory.ts index 02321229..0662b5e8 100644 --- a/src/database/services/save-memory.ts +++ b/src/database/services/save-memory.ts @@ -20,6 +20,7 @@ import { } from '@/database/support/save-policy' import { generateEmbeddingsForTargets } from '@/modules/embeddings/generate-target-embeddings' import HuggingFaceEmbeddingProvider from '@/modules/embeddings/hugging-face-embedding-provider' +import { scheduleMemoryMaintenance } from '@/modules/memory/background-maintenance' import contentHash from '@/support/content-hash' import type { EmbeddingProviderContract } from '@/types/embedding-provider' import type { ObservationKind, SaveResult } from '@/types/memory' @@ -46,6 +47,7 @@ export type SaveMemoriesInput = { export type SaveOptions = { embeddingProvider?: EmbeddingProviderContract + embeddingMode?: 'background' | 'disabled' | 'inline' projectUpdate?: { deletedFilePaths: string[] updatedFilePaths: string[] @@ -133,7 +135,7 @@ async function saveKonteksMemory( }) if (options.embedAfterSave !== false) { - await embedSavedTargets(options.embeddingProvider, createdAt, [ + await embedSavedTargets(_context, options, createdAt, [ { targetId: id, targetType: 'memory' }, ]) } @@ -182,7 +184,8 @@ export async function saveKonteksMemories( if (newTargets.length > 0) { await embedSavedTargets( - _options.embeddingProvider, + context, + _options, new Date().toISOString(), newTargets, ) @@ -260,7 +263,7 @@ export async function saveKonteksDiary( }) }) - await embedSavedTargets(options.embeddingProvider, createdAt, [ + await embedSavedTargets(_context, options, createdAt, [ { targetId: id, targetType: 'diary' }, ]) @@ -272,20 +275,39 @@ export async function saveKonteksDiary( } async function embedSavedTargets( - provider: EmbeddingProviderContract | undefined, + context: Project, + options: SaveOptions, createdAt: string, targets: Array<{ targetId: string; targetType: 'diary' | 'memory' }>, ): Promise { - if (targets.length === 0) { + if (targets.length === 0 || options.embeddingMode === 'disabled') { return } - try { + const embed = async () => { await generateEmbeddingsForTargets( - provider ?? new HuggingFaceEmbeddingProvider(), + options.embeddingProvider ?? new HuggingFaceEmbeddingProvider(), targets, createdAt, ) + } + + if (options.embeddingMode === 'background') { + scheduleMemoryMaintenance(context, { + metadata: { + targetCount: targets.length, + targetTypes: [ + ...new Set(targets.map(target => target.targetType)), + ], + }, + operation: embed, + operationName: 'embed_saved_targets', + }) + return + } + + try { + await embed() } catch { // Durable save succeeded; embedding is a best-effort retrieval index update. } diff --git a/src/modules/memory/background-maintenance.ts b/src/modules/memory/background-maintenance.ts new file mode 100644 index 00000000..b99bd55e --- /dev/null +++ b/src/modules/memory/background-maintenance.ts @@ -0,0 +1,72 @@ +import { withoutDatabaseTransactionContext } from '@/database/actions/_db' +import { withLoadedProjectContext } from '@/modules/project/context' +import { appendProjectErrorLog } from '@/support/error-log' +import type { LoadedProjectContext } from '@/types/project' + +type MemoryMaintenanceInput = { + dedupeKey?: string + metadata?: Record + operation: () => Promise + operationName: string +} + +declare global { + var __konteksWaitForMemoryMaintenanceForTests: + | (() => Promise) + | undefined +} + +const queues = new Map>() +const activeDedupeKeys = new Set() + +globalThis.__konteksWaitForMemoryMaintenanceForTests = async () => { + await Promise.all(queues.values()) +} + +export function scheduleMemoryMaintenance( + context: LoadedProjectContext, + input: MemoryMaintenanceInput, +): void { + const dedupeKey = input.dedupeKey + ? `${context.projectRoot}:${input.dedupeKey}` + : undefined + if (dedupeKey && activeDedupeKeys.has(dedupeKey)) { + return + } + if (dedupeKey) { + activeDedupeKeys.add(dedupeKey) + } + + const previous = queues.get(context.projectRoot) ?? Promise.resolve() + const queued = previous + .catch(() => undefined) + .then(() => + withoutDatabaseTransactionContext(() => + withLoadedProjectContext(context, () => + Promise.resolve().then(input.operation), + ), + ), + ) + .catch(async error => { + await withLoadedProjectContext(context, () => + appendProjectErrorLog({ + error, + metadata: { + ...input.metadata, + operation: input.operationName, + }, + surface: 'background_maintenance', + }), + ) + }) + + const tracked = queued.finally(() => { + if (queues.get(context.projectRoot) === tracked) { + queues.delete(context.projectRoot) + } + if (dedupeKey) { + activeDedupeKeys.delete(dedupeKey) + } + }) + queues.set(context.projectRoot, tracked) +} diff --git a/src/modules/memory/save-memory.ts b/src/modules/memory/save-memory.ts index a0775110..386d260f 100644 --- a/src/modules/memory/save-memory.ts +++ b/src/modules/memory/save-memory.ts @@ -7,9 +7,12 @@ import { saveKonteksMemories, } from '@/database/services/save-memory' import sharedEmbeddingProvider from '@/modules/embeddings/shared-embedding-provider' +import { scheduleMemoryMaintenance } from '@/modules/memory/background-maintenance' +import type { EmbeddingProviderContract } from '@/types/embedding-provider' import type { SaveResult } from '@/types/memory' import { loadMcpProjectContext, + type McpProjectContext, updateChangedProjectMemorySilently, } from './runtime' @@ -20,14 +23,12 @@ export async function saveMemories( const embeddingProvider = context.configExists ? sharedEmbeddingProvider() : undefined - const projectUpdate = await updateChangedProjectMemorySilently( - context, + const result = await saveKonteksMemories(context, input, { + embeddingMode: 'background', embeddingProvider, - ) - return await saveKonteksMemories(context, input, { - embeddingProvider, - projectUpdate, }) + scheduleChangedProjectMemoryUpdate(context, embeddingProvider) + return result } export async function saveDiary(input: SaveDiaryInput): Promise { @@ -35,12 +36,26 @@ export async function saveDiary(input: SaveDiaryInput): Promise { const embeddingProvider = context.configExists ? sharedEmbeddingProvider() : undefined - const projectUpdate = await updateChangedProjectMemorySilently( - context, - embeddingProvider, - ) - return await saveKonteksDiary(context, input, { + const result = await saveKonteksDiary(context, input, { + embeddingMode: 'background', embeddingProvider, - projectUpdate, + }) + scheduleChangedProjectMemoryUpdate(context, embeddingProvider) + return result +} + +function scheduleChangedProjectMemoryUpdate( + context: McpProjectContext, + embeddingProvider: EmbeddingProviderContract | undefined, +): void { + scheduleMemoryMaintenance(context, { + dedupeKey: 'changed_project_memory', + metadata: { + projectRoot: context.projectRoot, + }, + operation: async () => { + await updateChangedProjectMemorySilently(context, embeddingProvider) + }, + operationName: 'changed_project_memory', }) } diff --git a/src/modules/project/context.ts b/src/modules/project/context.ts index f9d62e21..9de5ca3e 100644 --- a/src/modules/project/context.ts +++ b/src/modules/project/context.ts @@ -1,3 +1,4 @@ +import { AsyncLocalStorage } from 'node:async_hooks' import { access, readFile, writeFile } from 'node:fs/promises' import { dirname, join, resolve } from 'node:path' import type { @@ -8,6 +9,10 @@ import type { export type { LoadedProjectContext } +const loadedProjectContext = new AsyncLocalStorage< + LoadedProjectContext | undefined +>() + export function createDefaultConfig(): KonteksConfig { return { extraction: { @@ -27,6 +32,15 @@ export async function writeProjectConfig( } export async function resolveProjectContext(): Promise { + const bound = loadedProjectContext.getStore() + if (bound) { + return { + configPath: bound.configPath, + memoryDir: bound.memoryDir, + projectRoot: bound.projectRoot, + } + } + const projectRoot = await findProjectRoot(process.cwd()) const memoryDir = join(projectRoot, '.konteks') @@ -38,6 +52,11 @@ export async function resolveProjectContext(): Promise { } export async function loadProjectContext(): Promise { + const bound = loadedProjectContext.getStore() + if (bound) { + return bound + } + const context = await resolveProjectContext() const config = await readConfig(context.configPath) @@ -48,6 +67,13 @@ export async function loadProjectContext(): Promise { } } +export function withLoadedProjectContext( + context: LoadedProjectContext, + operation: () => T, +): T { + return loadedProjectContext.run(context, operation) +} + async function findProjectRoot(start: string): Promise { let current = resolve(start) diff --git a/tests/features/memory/save-memory-embedding.test.ts b/tests/features/memory/save-memory-embedding.test.ts index 9f276ac9..228bd869 100644 --- a/tests/features/memory/save-memory-embedding.test.ts +++ b/tests/features/memory/save-memory-embedding.test.ts @@ -31,12 +31,39 @@ class ThrowingEmbeddingProvider implements EmbeddingProviderContract { } } +class DeferredEmbeddingProvider implements EmbeddingProviderContract { + public readonly dimensions = 8 + public readonly model = 'fake/deferred' + public readonly started: Promise + private releaseEmbedding!: () => void + private resolveStarted!: () => void + + public constructor() { + this.started = new Promise(resolve => { + this.resolveStarted = resolve + }) + } + + public release(): void { + this.releaseEmbedding() + } + + public async embed(texts: string[]): Promise { + this.resolveStarted() + await new Promise(resolve => { + this.releaseEmbedding = resolve + }) + return texts.map(() => new Float32Array([1, 0, 0, 0, 0, 0, 0, 0])) + } +} + beforeEach(() => { previousSqliteTestDatabase = process.env.KONTEKS_SQLITE_TEST_DATABASE process.env.KONTEKS_SQLITE_TEST_DATABASE = 'file' }) afterEach(async () => { + await globalThis.__konteksWaitForMemoryMaintenanceForTests?.() if (previousSqliteTestDatabase === undefined) { delete process.env.KONTEKS_SQLITE_TEST_DATABASE } else { @@ -134,8 +161,111 @@ describe('save memory embeddings', () => { expect(await embeddingRowsFor('memory')).toHaveLength(0) }) }) + + it('returns before background embedding finishes', async () => { + const projectRoot = await makeInitializedProject() + const provider = new DeferredEmbeddingProvider() + + await withProjectRoot(projectRoot, async () => { + const context = await loadProjectContext() + const save = saveKonteksMemories( + context, + { + memories: [ + { + content: + 'Background durable memory embedding should not block the save result.', + importance: 4, + kind: 'decision', + }, + ], + }, + { + embeddingMode: 'background', + embeddingProvider: provider, + }, + ) + await provider.started + const result = await raceSaveResult(save, () => provider.release()) + + const memoryId = result.memoryIds?.[0] + if (!memoryId) { + throw new Error('expected memory id') + } + await provider.started + expect(await rowsForObservation(memoryId)).toHaveLength(1) + expect(await embeddingRowsFor('memory')).toHaveLength(0) + + provider.release() + await globalThis.__konteksWaitForMemoryMaintenanceForTests?.() + await expectTargetIndexed(memoryId, 'memory', 'fake/deferred') + }) + }) + + it('runs background embedding against the saved project when cwd changes', async () => { + const sourceRoot = await makeInitializedProject() + const otherRoot = await makeInitializedProject() + const provider = new DeferredEmbeddingProvider() + + const save = withProjectRoot(sourceRoot, async () => { + const context = await loadProjectContext() + return await saveKonteksMemories( + context, + { + memories: [ + { + content: + 'Background embedding should keep the captured project context.', + importance: 4, + kind: 'constraint', + }, + ], + }, + { + embeddingMode: 'background', + embeddingProvider: provider, + }, + ) + }) + await provider.started + const result = await raceSaveResult(save, () => provider.release()) + + const memoryId = result.memoryIds?.[0] + if (!memoryId) { + throw new Error('expected memory id') + } + + await withProjectRoot(otherRoot, async () => { + provider.release() + await globalThis.__konteksWaitForMemoryMaintenanceForTests?.() + }) + + await withProjectRoot(sourceRoot, () => + expectTargetIndexed(memoryId, 'memory', 'fake/deferred'), + ) + await withProjectRoot(otherRoot, async () => { + expect(await embeddingRowsFor('memory')).toHaveLength(0) + }) + }) }) +async function raceSaveResult( + save: Promise>>, + onTimeout: () => void, +): Promise>> { + const result = await Promise.race([ + save, + new Promise<'timed-out'>(resolve => { + setTimeout(() => resolve('timed-out'), 50) + }), + ]) + if (result === 'timed-out') { + onTimeout() + } + expect(result).not.toBe('timed-out') + return result as Awaited> +} + describe('save diary embeddings', () => { it('embeds a newly saved diary retrieval document', async () => { const projectRoot = await makeInitializedProject() @@ -213,6 +343,7 @@ async function withProjectRoot( async function expectTargetIndexed( targetId: string, targetType: 'diary' | 'memory', + model = 'fake/all-MiniLM-L6-v2', ): Promise { const retrievalRows = await retrievalRowsFor(targetId, targetType) const embeddingRows = await embeddingRowsFor(targetType) @@ -220,7 +351,7 @@ async function expectTargetIndexed( expect(retrievalRows).toHaveLength(1) expect(embeddingRows).toHaveLength(1) expect(embeddingRows[0]).toMatchObject({ - model: 'fake/all-MiniLM-L6-v2', + model, targetId, targetType, }) diff --git a/tests/features/providers/protocol/retrieval-evals.test.ts b/tests/features/providers/protocol/retrieval-evals.test.ts index 1f886f0a..0ee1fe57 100644 --- a/tests/features/providers/protocol/retrieval-evals.test.ts +++ b/tests/features/providers/protocol/retrieval-evals.test.ts @@ -65,6 +65,7 @@ beforeEach(() => { }) afterEach(async () => { + await globalThis.__konteksWaitForMemoryMaintenanceForTests?.() await globalThis.__konteksWaitForVectorIndexRepairsForTests?.() globalThis.__konteksEmbeddingProviderForTests = undefined globalThis.__konteksVectorIndexConnectionFactoryForTests = undefined @@ -242,6 +243,7 @@ describe('retrieval quality evals', () => { 'Saved structured diary context after refreshing changed project memory.', }), ) + await globalThis.__konteksWaitForMemoryMaintenanceForTests?.() const text = extractText(result) const manifest = await readExtractionManifest(context.memoryDir) diff --git a/tests/features/providers/protocol/warm-up.test.ts b/tests/features/providers/protocol/warm-up.test.ts index 1542afce..1365f0e2 100644 --- a/tests/features/providers/protocol/warm-up.test.ts +++ b/tests/features/providers/protocol/warm-up.test.ts @@ -44,6 +44,7 @@ describe('konteks_warm_up', () => { }) afterEach(async () => { + await globalThis.__konteksWaitForMemoryMaintenanceForTests?.() if (previousDisableSharedEmbeddingProvider === undefined) { delete process.env.KONTEKS_DISABLE_SHARED_EMBEDDING_PROVIDER } else { From fe02db60d99184400fc681d62a45377292373d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=8D=95?= Date: Fri, 5 Jun 2026 09:52:05 +0800 Subject: [PATCH 2/2] feat: implement hard delete functionality for memory and diary retrieval artifacts --- docs/api/mcp.md | 2 + src/database/services/forget-memory.ts | 51 +++++ .../memory/durable-memory-graph.test.ts | 184 ++++++++++++++++++ 3 files changed, 237 insertions(+) diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 71cf97d7..ec21170b 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -41,6 +41,8 @@ MCP tools validate project health silently before doing work. If memory is not i Save tools return after durable memory is written. Changed-project refresh, embedding generation, and sqlite-vec indexing continue as best-effort background maintenance, so semantic retrieval can lag briefly after a save. Run `konteks rebuild` when you need an explicit full repair pass. +Forget modes differ by durability. `soft_delete` and `invalidate` hide or invalidate memory while preserving recoverable history; `hard_delete` physically removes the target from durable rows and retrieval/vector indexes. + Example durable memory payload: ```json diff --git a/src/database/services/forget-memory.ts b/src/database/services/forget-memory.ts index 862af97e..c850e075 100644 --- a/src/database/services/forget-memory.ts +++ b/src/database/services/forget-memory.ts @@ -1,5 +1,8 @@ import { randomUUID } from 'node:crypto' +import { and, eq } from 'drizzle-orm' +import getDb from '@/database/actions/_db' import appendMemoryEvent from '@/database/actions/append-memory-event' +import deleteRetrievalDocuments from '@/database/actions/delete-retrieval-documents' import hardDeleteForgetTarget from '@/database/actions/hard-delete-forget-target' import invalidateRelation from '@/database/actions/invalidate-relation' import markForgotten from '@/database/actions/mark-forgotten' @@ -7,9 +10,12 @@ import markSuppressed from '@/database/actions/mark-suppressed' import queryDiaries from '@/database/actions/query-diaries' import queryObservations from '@/database/actions/query-observations' import removeFromSearchIndex from '@/database/actions/remove-from-search-index' +import { targetEmbeddings } from '@/database/schema' import { deleteDurableTargetGraph } from '@/database/services/durable-memory-graph' import type { ForgetTarget, TargetKind } from '@/database/support/forget-target' +type RetrievalTargetType = 'diary' | 'memory' | 'section' + export type ForgetInput = { id?: string query?: string @@ -37,6 +43,9 @@ export default async function forgetMemory( } await removeFromSearchIndex(target.id) + if (mode === 'hard_delete') { + await deleteHardDeletedRetrievalArtifacts(target) + } if (target.kind === 'observation' || target.kind === 'diary_entry') { await deleteDurableTargetGraph(target.id) } @@ -111,6 +120,48 @@ async function applyForget( return markForgotten(target, reason) } +async function deleteHardDeletedRetrievalArtifacts( + target: ForgetTarget, +): Promise { + const targetType = retrievalTargetType(target.kind) + if (!targetType) { + return + } + + await deleteRetrievalDocuments(targetType, [target.id]) + await deleteTargetEmbeddings(targetType, target.id) +} + +async function deleteTargetEmbeddings( + targetType: RetrievalTargetType, + targetId: string, +): Promise { + const db = await getDb() + await db + .delete(targetEmbeddings) + .where( + and( + eq(targetEmbeddings.targetType, targetType), + eq(targetEmbeddings.targetId, targetId), + ), + ) +} + +function retrievalTargetType( + targetKind: TargetKind, +): RetrievalTargetType | undefined { + if (targetKind === 'diary_entry') { + return 'diary' + } + if (targetKind === 'observation') { + return 'memory' + } + if (targetKind === 'section') { + return 'section' + } + return undefined +} + function inferKind(id: string): TargetKind { if (id.startsWith('section_')) { return 'section' diff --git a/tests/features/memory/durable-memory-graph.test.ts b/tests/features/memory/durable-memory-graph.test.ts index 89714c0a..f80905fe 100644 --- a/tests/features/memory/durable-memory-graph.test.ts +++ b/tests/features/memory/durable-memory-graph.test.ts @@ -2,14 +2,25 @@ import { afterEach, describe, expect, it } from 'bun:test' import { mkdtemp, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { and, eq } from 'drizzle-orm' +import getDb from '@/database/actions/_db' import historicalRelations from '@/database/actions/historical-relations' import traverseNeighbors from '@/database/actions/traverse-neighbors' +import { + diaryEntries, + observations, + retrievalDocuments, + retrievalDocumentsFts, + targetEmbeddings, + vectorIndexEntries, +} from '@/database/schema' import forgetMemory from '@/database/services/forget-memory' import { entityIdFor } from '@/database/services/graph' import { saveKonteksDiary, saveKonteksMemories, } from '@/database/services/save-memory' +import searchMemory from '@/database/services/search-memory' import { extractProject } from '@/modules/extraction/extract-project' import recallRepositoryMemory from '@/modules/memory/recall-repository-memory' import { loadProjectContext } from '@/modules/project/context' @@ -184,6 +195,76 @@ describe('durable memory graph projection', () => { ).resolves.toEqual([]) }) + it('removes durable memory retrieval and vector artifacts when hard deleted', async () => { + const projectRoot = await makeExtractedProject() + process.chdir(projectRoot) + const context = await loadProjectContext() + const phrase = + 'Hard delete should remove durable memory retrieval vector artifacts.' + const result = await saveKonteksMemories( + context, + { + memories: [ + { + content: phrase, + importance: 4, + kind: 'constraint', + }, + ], + }, + { embeddingProvider: new FakeEmbeddingProvider() }, + ) + const memoryId = result.memoryIds?.[0] + if (!memoryId) { + throw new Error('expected memory id') + } + + await expectRetrievalArtifacts(memoryId, 'memory', 1) + + await forgetMemory({ + id: memoryId, + mode: 'hard_delete', + reason: 'hard delete cleanup test', + }) + + await expect(rowsForObservation(memoryId)).resolves.toHaveLength(0) + await expectRetrievalArtifacts(memoryId, 'memory', 0) + await expectSearchToOmit(phrase) + }) + + it('removes diary retrieval and vector artifacts when hard deleted', async () => { + const projectRoot = await makeExtractedProject() + process.chdir(projectRoot) + const context = await loadProjectContext() + const phrase = + 'Hard delete should remove durable diary retrieval vector artifacts.' + const diary = await saveKonteksDiary( + context, + { + subject: 'hard delete diary cleanup', + summary: phrase, + tags: ['forget'], + }, + { embeddingProvider: new FakeEmbeddingProvider() }, + ) + const diaryId = diary.diaryId + if (!diaryId) { + throw new Error('expected diary id') + } + + await expectRetrievalArtifacts(diaryId, 'diary', 1) + + await forgetMemory({ + id: diaryId, + mode: 'hard_delete', + reason: 'hard delete diary cleanup test', + }) + + await expect(rowsForDiary(diaryId)).resolves.toHaveLength(0) + await expectRetrievalArtifacts(diaryId, 'diary', 0) + await expectSearchToOmit(phrase) + }) + it('supersedes prior decision graph claims and recalls them as history', async () => { const projectRoot = await makeExtractedProject() process.chdir(projectRoot) @@ -397,3 +478,106 @@ async function makeExtractedProject(): Promise { return projectRoot } + +async function expectRetrievalArtifacts( + targetId: string, + targetType: 'diary' | 'memory' | 'section', + expectedCount: number, +): Promise { + await expect(retrievalRowsFor(targetId, targetType)).resolves.toHaveLength( + expectedCount, + ) + await expect( + retrievalFtsRowsFor(targetId, targetType), + ).resolves.toHaveLength(expectedCount) + await expect(embeddingRowsFor(targetId, targetType)).resolves.toHaveLength( + expectedCount, + ) + await expect( + vectorIndexRowsFor(targetId, targetType), + ).resolves.toHaveLength(expectedCount) +} + +async function retrievalRowsFor( + targetId: string, + targetType: 'diary' | 'memory' | 'section', +) { + const db = await getDb() + return await db + .select() + .from(retrievalDocuments) + .where( + and( + eq(retrievalDocuments.targetId, targetId), + eq(retrievalDocuments.targetType, targetType), + ), + ) +} + +async function retrievalFtsRowsFor( + targetId: string, + targetType: 'diary' | 'memory' | 'section', +) { + const db = await getDb() + return await db + .select() + .from(retrievalDocumentsFts) + .where( + and( + eq(retrievalDocumentsFts.targetId, targetId), + eq(retrievalDocumentsFts.targetType, targetType), + ), + ) +} + +async function embeddingRowsFor( + targetId: string, + targetType: 'diary' | 'memory' | 'section', +) { + const db = await getDb() + return await db + .select() + .from(targetEmbeddings) + .where( + and( + eq(targetEmbeddings.targetId, targetId), + eq(targetEmbeddings.targetType, targetType), + ), + ) +} + +async function vectorIndexRowsFor( + targetId: string, + targetType: 'diary' | 'memory' | 'section', +) { + const db = await getDb() + return await db + .select() + .from(vectorIndexEntries) + .where( + and( + eq(vectorIndexEntries.targetId, targetId), + eq(vectorIndexEntries.targetType, targetType), + ), + ) +} + +async function rowsForObservation(id: string) { + const db = await getDb() + return await db.select().from(observations).where(eq(observations.id, id)) +} + +async function rowsForDiary(id: string) { + const db = await getDb() + return await db.select().from(diaryEntries).where(eq(diaryEntries.id, id)) +} + +async function expectSearchToOmit(phrase: string): Promise { + const results = await searchMemory( + { limit: 10, query: phrase }, + { embeddingProvider: new FakeEmbeddingProvider() }, + ) + expect(results.map(result => result.excerpt).join('\n')).not.toContain( + phrase, + ) +}