Implement hard delete and background memory maintenance

docs/api/mcp.md

@@ -39,6 +39,10 @@ Tools are lower-level callable operations used by agents and debugging workflows
 
 MCP tools validate project health silently before doing work. If memory is not initialized or a derived-memory rebuild is required, the tool fails with a short actionable error instead of returning status context.
 
+Save tools return after durable memory is written. Changed-project refresh, embedding generation, and sqlite-vec indexing continue as best-effort background maintenance, so semantic retrieval can lag briefly after a save. Run `konteks rebuild` when you need an explicit full repair pass.
+
+Forget modes differ by durability. `soft_delete` and `invalidate` hide or invalidate memory while preserving recoverable history; `hard_delete` physically removes the target from durable rows and retrieval/vector indexes.
+
 Example durable memory payload:
 
 ```json

docs/getting-started/lifecycle.md

@@ -66,6 +66,8 @@ Use the `/konteks-save` prompt to persist the outcome of the current agent sessi
 
 A single session can contain one task or several related tasks. The agent saves compact structured durable memories first, then writes one session diary summarizing the outcome. Durable memories should be future-useful rules, decisions, constraints, preferences, blockers, or code insights. The diary should be a compact summary of the task, outcome, verification, unresolved questions, and exact next steps.
 
+Save confirms durable persistence immediately. Project refresh and vector indexing may finish shortly afterward in background maintenance.
+
 > [!TIP]
 > Recommendation: Prefer saving when the session is complete or about to be closed. If progress is partial, the session diary should include pending items and exact next steps.
 

src/database/services/forget-memory.ts

@@ -1,15 +1,21 @@
 import { randomUUID } from 'node:crypto'
+import { and, eq } from 'drizzle-orm'
+import getDb from '@/database/actions/_db'
 import appendMemoryEvent from '@/database/actions/append-memory-event'
+import deleteRetrievalDocuments from '@/database/actions/delete-retrieval-documents'
 import hardDeleteForgetTarget from '@/database/actions/hard-delete-forget-target'
 import invalidateRelation from '@/database/actions/invalidate-relation'
 import markForgotten from '@/database/actions/mark-forgotten'
 import markSuppressed from '@/database/actions/mark-suppressed'
 import queryDiaries from '@/database/actions/query-diaries'
 import queryObservations from '@/database/actions/query-observations'
 import removeFromSearchIndex from '@/database/actions/remove-from-search-index'
+import { targetEmbeddings } from '@/database/schema'
 import { deleteDurableTargetGraph } from '@/database/services/durable-memory-graph'
 import type { ForgetTarget, TargetKind } from '@/database/support/forget-target'
 
+type RetrievalTargetType = 'diary' | 'memory' | 'section'
+
 export type ForgetInput = {
     id?: string
     query?: string
@@ -37,6 +43,9 @@ export default async function forgetMemory(
         }
 
         await removeFromSearchIndex(target.id)
+        if (mode === 'hard_delete') {
+            await deleteHardDeletedRetrievalArtifacts(target)
+        }
         if (target.kind === 'observation' || target.kind === 'diary_entry') {
             await deleteDurableTargetGraph(target.id)
         }
@@ -111,6 +120,48 @@ async function applyForget(
     return markForgotten(target, reason)
 }
 
+async function deleteHardDeletedRetrievalArtifacts(
+    target: ForgetTarget,
+): Promise<void> {
+    const targetType = retrievalTargetType(target.kind)
+    if (!targetType) {
+        return
+    }
+
+    await deleteRetrievalDocuments(targetType, [target.id])
+    await deleteTargetEmbeddings(targetType, target.id)
+}
+
+async function deleteTargetEmbeddings(
+    targetType: RetrievalTargetType,
+    targetId: string,
+): Promise<void> {
+    const db = await getDb()
+    await db
+        .delete(targetEmbeddings)
+        .where(
+            and(
+                eq(targetEmbeddings.targetType, targetType),
+                eq(targetEmbeddings.targetId, targetId),
+            ),
+        )
+}
+
+function retrievalTargetType(
+    targetKind: TargetKind,
+): RetrievalTargetType | undefined {
+    if (targetKind === 'diary_entry') {
+        return 'diary'
+    }
+    if (targetKind === 'observation') {
+        return 'memory'
+    }
+    if (targetKind === 'section') {
+        return 'section'
+    }
+    return undefined
+}
+
 function inferKind(id: string): TargetKind {
     if (id.startsWith('section_')) {
         return 'section'

src/database/services/save-memory.ts

@@ -20,6 +20,7 @@ import {
 } from '@/database/support/save-policy'
 import { generateEmbeddingsForTargets } from '@/modules/embeddings/generate-target-embeddings'
 import HuggingFaceEmbeddingProvider from '@/modules/embeddings/hugging-face-embedding-provider'
+import { scheduleMemoryMaintenance } from '@/modules/memory/background-maintenance'
 import contentHash from '@/support/content-hash'
 import type { EmbeddingProviderContract } from '@/types/embedding-provider'
 import type { ObservationKind, SaveResult } from '@/types/memory'
@@ -46,6 +47,7 @@ export type SaveMemoriesInput = {
 
 export type SaveOptions = {
     embeddingProvider?: EmbeddingProviderContract
+    embeddingMode?: 'background' | 'disabled' | 'inline'
     projectUpdate?: {
         deletedFilePaths: string[]
         updatedFilePaths: string[]
@@ -133,7 +135,7 @@ async function saveKonteksMemory(
     })
 
     if (options.embedAfterSave !== false) {
-        await embedSavedTargets(options.embeddingProvider, createdAt, [
+        await embedSavedTargets(_context, options, createdAt, [
             { targetId: id, targetType: 'memory' },
         ])
     }
@@ -182,7 +184,8 @@ export async function saveKonteksMemories(
 
     if (newTargets.length > 0) {
         await embedSavedTargets(
-            _options.embeddingProvider,
+            context,
+            _options,
             new Date().toISOString(),
             newTargets,
         )
@@ -260,7 +263,7 @@ export async function saveKonteksDiary(
         })
     })
 
-    await embedSavedTargets(options.embeddingProvider, createdAt, [
+    await embedSavedTargets(_context, options, createdAt, [
         { targetId: id, targetType: 'diary' },
     ])
 
@@ -272,20 +275,39 @@ export async function saveKonteksDiary(
 }
 
 async function embedSavedTargets(
-    provider: EmbeddingProviderContract | undefined,
+    context: Project,
+    options: SaveOptions,
     createdAt: string,
     targets: Array<{ targetId: string; targetType: 'diary' | 'memory' }>,
 ): Promise<void> {
-    if (targets.length === 0) {
+    if (targets.length === 0 || options.embeddingMode === 'disabled') {
         return
     }
 
-    try {
+    const embed = async () => {
         await generateEmbeddingsForTargets(
-            provider ?? new HuggingFaceEmbeddingProvider(),
+            options.embeddingProvider ?? new HuggingFaceEmbeddingProvider(),
             targets,
             createdAt,
         )
+    }
+
+    if (options.embeddingMode === 'background') {
+        scheduleMemoryMaintenance(context, {
+            metadata: {
+                targetCount: targets.length,
+                targetTypes: [
+                    ...new Set(targets.map(target => target.targetType)),
+                ],
+            },
+            operation: embed,
+            operationName: 'embed_saved_targets',
+        })
+        return
+    }
+
+    try {
+        await embed()
     } catch {
         // Durable save succeeded; embedding is a best-effort retrieval index update.
     }

src/modules/memory/background-maintenance.ts

@@ -0,0 +1,72 @@
+import { withoutDatabaseTransactionContext } from '@/database/actions/_db'
+import { withLoadedProjectContext } from '@/modules/project/context'
+import { appendProjectErrorLog } from '@/support/error-log'
+import type { LoadedProjectContext } from '@/types/project'
+
+type MemoryMaintenanceInput = {
+    dedupeKey?: string
+    metadata?: Record<string, unknown>
+    operation: () => Promise<void>
+    operationName: string
+}
+
+declare global {
+    var __konteksWaitForMemoryMaintenanceForTests:
+        | (() => Promise<void>)
+        | undefined
+}
+
+const queues = new Map<string, Promise<void>>()
+const activeDedupeKeys = new Set<string>()
+
+globalThis.__konteksWaitForMemoryMaintenanceForTests = async () => {
+    await Promise.all(queues.values())
+}
+
+export function scheduleMemoryMaintenance(
+    context: LoadedProjectContext,
+    input: MemoryMaintenanceInput,
+): void {
+    const dedupeKey = input.dedupeKey
+        ? `${context.projectRoot}:${input.dedupeKey}`
+        : undefined
+    if (dedupeKey && activeDedupeKeys.has(dedupeKey)) {
+        return
+    }
+    if (dedupeKey) {
+        activeDedupeKeys.add(dedupeKey)
+    }
+
+    const previous = queues.get(context.projectRoot) ?? Promise.resolve()
+    const queued = previous
+        .catch(() => undefined)
+        .then(() =>
+            withoutDatabaseTransactionContext(() =>
+                withLoadedProjectContext(context, () =>
+                    Promise.resolve().then(input.operation),
+                ),
+            ),
+        )
+        .catch(async error => {
+            await withLoadedProjectContext(context, () =>
+                appendProjectErrorLog({
+                    error,
+                    metadata: {
+                        ...input.metadata,
+                        operation: input.operationName,
+                    },
+                    surface: 'background_maintenance',
+                }),
+            )
+        })
+
+    const tracked = queued.finally(() => {
+        if (queues.get(context.projectRoot) === tracked) {
+            queues.delete(context.projectRoot)
+        }
+        if (dedupeKey) {
+            activeDedupeKeys.delete(dedupeKey)
+        }
+    })
+    queues.set(context.projectRoot, tracked)
+}

src/modules/memory/save-memory.ts

@@ -7,9 +7,12 @@ import {
     saveKonteksMemories,
 } from '@/database/services/save-memory'
 import sharedEmbeddingProvider from '@/modules/embeddings/shared-embedding-provider'
+import { scheduleMemoryMaintenance } from '@/modules/memory/background-maintenance'
+import type { EmbeddingProviderContract } from '@/types/embedding-provider'
 import type { SaveResult } from '@/types/memory'
 import {
     loadMcpProjectContext,
+    type McpProjectContext,
     updateChangedProjectMemorySilently,
 } from './runtime'
 
@@ -20,27 +23,39 @@ export async function saveMemories(
     const embeddingProvider = context.configExists
         ? sharedEmbeddingProvider()
         : undefined
-    const projectUpdate = await updateChangedProjectMemorySilently(
-        context,
+    const result = await saveKonteksMemories(context, input, {
+        embeddingMode: 'background',
         embeddingProvider,
-    )
-    return await saveKonteksMemories(context, input, {
-        embeddingProvider,
-        projectUpdate,
     })
+    scheduleChangedProjectMemoryUpdate(context, embeddingProvider)
+    return result
 }
 
 export async function saveDiary(input: SaveDiaryInput): Promise<SaveResult> {
     const context = await loadMcpProjectContext()
     const embeddingProvider = context.configExists
         ? sharedEmbeddingProvider()
         : undefined
-    const projectUpdate = await updateChangedProjectMemorySilently(
-        context,
-        embeddingProvider,
-    )
-    return await saveKonteksDiary(context, input, {
+    const result = await saveKonteksDiary(context, input, {
+        embeddingMode: 'background',
         embeddingProvider,
-        projectUpdate,
+    })
+    scheduleChangedProjectMemoryUpdate(context, embeddingProvider)
+    return result
+}
+
+function scheduleChangedProjectMemoryUpdate(
+    context: McpProjectContext,
+    embeddingProvider: EmbeddingProviderContract | undefined,
+): void {
+    scheduleMemoryMaintenance(context, {
+        dedupeKey: 'changed_project_memory',
+        metadata: {
+            projectRoot: context.projectRoot,
+        },
+        operation: async () => {
+            await updateChangedProjectMemorySilently(context, embeddingProvider)
+        },
+        operationName: 'changed_project_memory',
     })
 }

src/modules/project/context.ts

@@ -1,3 +1,4 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
 import { access, readFile, writeFile } from 'node:fs/promises'
 import { dirname, join, resolve } from 'node:path'
 import type {
@@ -8,6 +9,10 @@ import type {
 
 export type { LoadedProjectContext }
 
+const loadedProjectContext = new AsyncLocalStorage<
+    LoadedProjectContext | undefined
+>()
+
 export function createDefaultConfig(): KonteksConfig {
     return {
         extraction: {
@@ -27,6 +32,15 @@ export async function writeProjectConfig(
 }
 
 export async function resolveProjectContext(): Promise<ProjectContext> {
+    const bound = loadedProjectContext.getStore()
+    if (bound) {
+        return {
+            configPath: bound.configPath,
+            memoryDir: bound.memoryDir,
+            projectRoot: bound.projectRoot,
+        }
+    }
+
     const projectRoot = await findProjectRoot(process.cwd())
     const memoryDir = join(projectRoot, '.konteks')
 
@@ -38,6 +52,11 @@ export async function resolveProjectContext(): Promise<ProjectContext> {
 }
 
 export async function loadProjectContext(): Promise<LoadedProjectContext> {
+    const bound = loadedProjectContext.getStore()
+    if (bound) {
+        return bound
+    }
+
     const context = await resolveProjectContext()
     const config = await readConfig(context.configPath)
 
@@ -48,6 +67,13 @@ export async function loadProjectContext(): Promise<LoadedProjectContext> {
     }
 }
 
+export function withLoadedProjectContext<T>(
+    context: LoadedProjectContext,
+    operation: () => T,
+): T {
+    return loadedProjectContext.run(context, operation)
+}
+
 async function findProjectRoot(start: string): Promise<string> {
     let current = resolve(start)