diff --git a/.env.example b/.env.example index 4b9deb0b60..60d201e56a 100644 --- a/.env.example +++ b/.env.example @@ -130,6 +130,9 @@ # AGENTMEMORY_DROP_STALE_INDEX=true # Drop on-disk BM25 / vector index on startup if dim guard fires (#248). Recovery toggle for stuck-state debugging. # AGENTMEMORY_IMAGE_EMBEDDINGS=true # Enable image embeddings when an image provider is present (experimental). # AGENTMEMORY_INDEX_SAVE_INTERVAL_MS=600000 # Minimum time between search index saves; shutdown and deletes still save immediately. +# AGENTMEMORY_VECTOR_BUCKET_SIZE=500 # Vectors held per storage bucket before a new bucket opens; a save only rewrites buckets it changed. +# AGENTMEMORY_VECTOR_BACKFILL_MAX=500 # Cap on vectors embedded per boot to backfill ones missing since the last save. +# AGENTMEMORY_VECTOR_BACKFILL=all # Required to backfill an entire store's vectors at once (no prior vector save, or a stale legacy manifest); unset means that case waits for an explicit opt-in. # ----------------------------------------------------------------------------- # 6. CLI / runtime knobs diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index 5fe071567b..44e300d1df 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 44 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 47 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -46,6 +46,9 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_TOOLS` - `AGENTMEMORY_URL` - `AGENTMEMORY_USE_DOCKER` +- `AGENTMEMORY_VECTOR_BACKFILL` +- `AGENTMEMORY_VECTOR_BACKFILL_MAX` +- `AGENTMEMORY_VECTOR_BUCKET_SIZE` - `AGENTMEMORY_VERBOSE` - `AGENTMEMORY_VIEWER_HOST` - `AGENTMEMORY_VIEWER_URL` diff --git a/src/config.ts b/src/config.ts index ac82bd445f..a7986f9604 100644 --- a/src/config.ts +++ b/src/config.ts @@ -477,6 +477,24 @@ export function getIndexSaveIntervalMs(): number { return raw > 0 ? raw : INDEX_SAVE_INTERVAL_DEFAULT_MS; } +export const VECTOR_BUCKET_SIZE_DEFAULT = 500; + +export function getVectorBucketSize(): number { + const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BUCKET_SIZE"], VECTOR_BUCKET_SIZE_DEFAULT); + return raw > 0 ? raw : VECTOR_BUCKET_SIZE_DEFAULT; +} + +export const VECTOR_BACKFILL_MAX_DEFAULT = 500; + +export function getVectorBackfillMax(): number { + const raw = safeParseInt(getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL_MAX"], VECTOR_BACKFILL_MAX_DEFAULT); + return raw > 0 ? raw : VECTOR_BACKFILL_MAX_DEFAULT; +} + +export function isVectorBackfillAllEnabled(): boolean { + return getMergedEnv()["AGENTMEMORY_VECTOR_BACKFILL"] === "all"; +} + export function isStandaloneMcp(): boolean { return getMergedEnv()["STANDALONE_MCP"] === "true"; } diff --git a/src/functions/search.ts b/src/functions/search.ts index 8abce07f45..2d2f414785 100644 --- a/src/functions/search.ts +++ b/src/functions/search.ts @@ -9,7 +9,12 @@ import type { EmbeddingProvider } from '../types.js' import { memoryToObservation } from '../state/memory-utils.js' import { recordAccessBatch } from './access-tracker.js' import { logger } from "../logger.js"; -import { getAgentId, isAgentScopeIsolated } from "../config.js"; +import { + getAgentId, + getVectorBackfillMax, + isAgentScopeIsolated, + isVectorBackfillAllEnabled, +} from "../config.js"; let index: SearchIndex | null = null let vectorIndex: VectorIndex | null = null @@ -45,6 +50,19 @@ export function isMemoryIndexReady(): boolean { return memoryIndexReady } +let bm25RebuildIncomplete = false +export function isBm25RebuildIncomplete(): boolean { + return bm25RebuildIncomplete +} + +let pendingVectorBackfill = 0 +export function getPendingVectorBackfillCount(): number { + return pendingVectorBackfill +} +export function setPendingVectorBackfillCount(n: number): void { + pendingVectorBackfill = Math.max(0, n) +} + export function getSearchIndex(): SearchIndex { if (!index) index = new SearchIndex() return index @@ -399,6 +417,146 @@ export async function rebuildIndex(kv: StateKV): Promise { return indexed } +export type VectorBackfillJob = { + id: string + sessionId: string + text: string + context: { kind: "memory" | "observation" | "synthetic"; logId: string } +} + +const SESSIONS_LIST_RETRY_ATTEMPTS = 3 +const SESSIONS_LIST_RETRY_BASE_MS = 250 + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +async function listSessionsWithRetry(kv: StateKV): Promise { + let lastErr: unknown + for (let attempt = 0; attempt < SESSIONS_LIST_RETRY_ATTEMPTS; attempt++) { + try { + return await kv.list(KV.sessions) + } catch (err) { + lastErr = err + if (attempt < SESSIONS_LIST_RETRY_ATTEMPTS - 1) { + await delay(SESSIONS_LIST_RETRY_BASE_MS * (attempt + 1)) + } + } + } + logger.warn('rebuildKeywordIndex: failed to load sessions after retries', { + attempts: SESSIONS_LIST_RETRY_ATTEMPTS, + error: lastErr instanceof Error ? lastErr.message : String(lastErr), + }) + return null +} + +export async function rebuildKeywordIndex( + kv: StateKV, + vectorBackfillSince?: string | null, +): Promise<{ documents: number; vectorJobs: VectorBackfillJob[]; fullBackfillPending: number }> { + const idx = getSearchIndex() + idx.clear() + memoryIndexReady = false + bm25RebuildIncomplete = false + const vi = vectorIndex + const backfillEligible = Boolean(vi && currentEmbeddingProvider) && vectorBackfillSince !== undefined + const wholeStoreBackfill = vectorBackfillSince === null + const backfillGated = backfillEligible && wholeStoreBackfill && !isVectorBackfillAllEnabled() + const backfillActive = backfillEligible && !backfillGated + const cutoff = typeof vectorBackfillSince === 'string' ? Date.parse(vectorBackfillSince) : Number.NaN + const backfillCap = backfillActive ? getVectorBackfillMax() : 0 + const vectorJobs: VectorBackfillJob[] = [] + let fullBackfillPending = 0 + const consider = ( + id: string, + sessionId: string, + text: string, + timestamp: string | undefined, + kind: "memory" | "observation", + ): void => { + if (!backfillEligible || vi?.has(id)) return + if (!Number.isNaN(cutoff) && !(Date.parse(timestamp ?? '') > cutoff)) return + if (backfillGated) { + fullBackfillPending++ + return + } + if (!backfillActive || vectorJobs.length >= backfillCap) return + vectorJobs.push({ id, sessionId, text, context: { kind, logId: id } }) + } + + let documents = 0 + let memoriesLoaded = false + try { + const memories = await kv.list(KV.memories) + memoriesLoaded = true + for (const memory of memories) { + if (memory.isLatest === false) continue + if (!memory.title || !memory.content) continue + idx.add(memoryToObservation(memory)) + consider(memory.id, memory.sessionIds?.[0] ?? 'memory', memory.title + ' ' + memory.content, memory.createdAt, "memory") + documents++ + } + } catch (err) { + logger.warn('rebuildKeywordIndex: failed to load memories', { + error: err instanceof Error ? err.message : String(err), + }) + } + + const sessions = await listSessionsWithRetry(kv) + if (sessions === null) { + bm25RebuildIncomplete = true + if (memoriesLoaded) memoryIndexReady = true + return { documents, vectorJobs, fullBackfillPending } + } + const failedSessions: string[] = [] + for (let batch = 0; batch < sessions.length; batch += 10) { + const chunk = sessions.slice(batch, batch + 10) + const results = await Promise.all( + chunk.map(async (s) => { + try { + return await kv.list(KV.observations(s.id)) + } catch { + failedSessions.push(s.id) + return [] as CompressedObservation[] + } + }) + ) + for (const obs of results.flat()) { + if (!obs.title || !obs.narrative) continue + idx.add(obs) + consider(obs.id, obs.sessionId, obs.title + ' ' + obs.narrative, obs.timestamp, "observation") + documents++ + } + } + if (failedSessions.length > 0) { + bm25RebuildIncomplete = true + logger.warn('rebuildKeywordIndex: failed to load observations for sessions', { failedSessions }) + } + if (memoriesLoaded) memoryIndexReady = true + return { documents, vectorJobs, fullBackfillPending } +} + +const BACKFILL_SAVE_EVERY_BATCHES = 10 + +export async function backfillVectors(jobs: VectorBackfillJob[]): Promise { + const batchSize = getRebuildEmbedBatchSize() + let added = 0 + let batchesSinceSave = 0 + setPendingVectorBackfillCount(jobs.length) + for (let offset = 0; offset < jobs.length; offset += batchSize) { + const { ok } = await vectorIndexAddBatchGuarded(jobs.slice(offset, offset + batchSize)) + added += ok + setPendingVectorBackfillCount(jobs.length - Math.min(jobs.length, offset + batchSize)) + batchesSinceSave++ + if (batchesSinceSave >= BACKFILL_SAVE_EVERY_BATCHES) { + await flushIndexSave() + batchesSinceSave = 0 + } + } + if (added > 0) await flushIndexSave() + return added +} + export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void { sdk.registerFunction( 'mem::search', @@ -480,10 +638,10 @@ export function registerSearchFunction(sdk: IIIClient, kv: StateKV): void { // Share one rebuild across concurrent cold-start queries so they // don't each walk the whole corpus and saturate the pool. if (!rebuildPromise) { - rebuildPromise = rebuildIndex(kv) - .then((count) => { - logger.info('Search index rebuilt', { entries: count }) - return count + rebuildPromise = rebuildKeywordIndex(kv) + .then((result) => { + logger.info('Search index rebuilt', { entries: result.documents }) + return result.documents }) .catch((err) => { logger.warn('Index rebuild failed', { diff --git a/src/functions/status.ts b/src/functions/status.ts index 0b3ab5310a..c6577c44e6 100644 --- a/src/functions/status.ts +++ b/src/functions/status.ts @@ -1,4 +1,4 @@ -import { V8_MAX_STRING_CHARS, type IndexLegStatus, type IndexPersistenceStatus } from "../state/index-persistence.js"; +import type { IndexLegStatus, IndexPersistenceStatus } from "../state/index-persistence.js"; export type StatusLevel = "ok" | "info" | "warn" | "error"; @@ -57,6 +57,8 @@ export interface StatusInputs { observationsIndexed: number; missingObservations: number | null; sessions: number | null; + bm25Incomplete: boolean; + pendingVectorBackfill: number; }; graph: GraphStatsInput | null; graphExtractionEnabled: boolean; @@ -85,7 +87,6 @@ export interface StatusReport { const FAILURE_RATE_THRESHOLD = 0.2; const FAILURE_MIN_CALLS = 5; const GRAPH_SNAPSHOT_STALE_SECONDS = 24 * 60 * 60; -const VECTOR_STRING_WARN_RATIO = 0.8; const LEVEL_RANK: Record = { ok: 0, info: 1, warn: 2, error: 3 }; const FUNCTION_FIXES: Record = { @@ -173,42 +174,54 @@ export function evaluateStatus(input: StatusInputs): StatusReport { }); } + if (input.index.bm25Incomplete) { + problems.push({ + level: "error", + code: "bm25-rebuild-incomplete", + message: "The keyword index could not load every session at boot, so some observations are missing from search until the next successful rebuild.", + fix: "Check the server log for the session listing failure. Restarting or the next cold-start search retries automatically.", + }); + } + + if (input.index.pendingVectorBackfill > 0) { + problems.push({ + level: "info", + code: "index-vector-backfill-pending", + message: `${input.index.pendingVectorBackfill} documents are waiting for a vector embedding.`, + fix: "This runs in the background and is capped per boot by AGENTMEMORY_VECTOR_BACKFILL_MAX. If it is stuck at a nonzero count with no embedding provider errors, set AGENTMEMORY_VECTOR_BACKFILL=all to run a full backfill.", + }); + } + const persistence = input.indexPersistence ?? null; - if (persistence) { - const legs: Array<[string, IndexLegStatus | null]> = [ - ["BM25", persistence.bm25], - ["vector", persistence.vector], - ]; - for (const [label, leg] of legs) { - if (!leg) continue; - if (leg.lastError) { - problems.push({ - level: "error", - code: "index-save-failing", - message: `The ${label} search index could not be saved: ${leg.lastError}. Search keeps working from memory, but changes since the last save are lost on restart.`, - fix: "Check the server log for the failing state write. The next save retries automatically.", - }); - } - const dirtyAge = secondsBetween(input.now, leg.dirtySince ?? undefined); - if (!leg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) { - problems.push({ - level: "warn", - code: "index-save-stale", - message: `The ${label} search index has unsaved changes from ${formatDuration(dirtyAge)} ago.`, - fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.", - }); - } + const vectorLeg = persistence?.vector ?? null; + if (persistence && vectorLeg) { + if (vectorLeg.lastError) { + problems.push({ + level: "error", + code: "index-save-failing", + message: `The vector index could not be saved: ${vectorLeg.lastError}. Search keeps working from memory, but vectors added since the last save are lost on restart.`, + fix: "Check the server log for the failing state write. The next save retries automatically.", + }); } - const vectorChars = persistence.vector?.serializedChars ?? null; - if (vectorChars !== null && vectorChars > V8_MAX_STRING_CHARS * VECTOR_STRING_WARN_RATIO) { + const dirtyAge = secondsBetween(input.now, vectorLeg.dirtySince ?? undefined); + if (!vectorLeg.lastError && dirtyAge !== null && dirtyAge * 1000 > 2 * persistence.saveIntervalMs) { problems.push({ level: "warn", - code: "vector-index-near-string-limit", - message: `The saved vector index is ${Math.round((vectorChars / V8_MAX_STRING_CHARS) * 100)}% of the largest string Node.js can hold. Past that limit it can no longer be saved or loaded.`, - fix: "Upgrade agentmemory once bucketed vector storage ships, or reduce the index (a smaller embedding model, or forget unused observations) before it reaches the limit.", + code: "index-save-stale", + message: `The vector index has unsaved changes from ${formatDuration(dirtyAge)} ago.`, + fix: "Saves run at most once per AGENTMEMORY_INDEX_SAVE_INTERVAL_MS. Check the server log for save errors or a save that never finishes.", }); } } + if (persistence?.vectorCountShortfall) { + const { expected, loaded } = persistence.vectorCountShortfall; + problems.push({ + level: "warn", + code: "index-vector-count-shortfall", + message: `Only ${loaded} of ${expected} vectors loaded from the last save; a bounded backfill re-embeds the rest.`, + fix: "No action needed — missing vectors are queued for backfill up to the per-boot cap.", + }); + } let graph: StatusReport["graph"] = null; if (input.graph) { @@ -300,16 +313,13 @@ function legSummary(report: StatusReport, leg: IndexLegStatus): string { function indexPersistenceRows(report: StatusReport): string { const persistence = report.indexPersistence; if (!persistence) return ""; - let rows = row("BM25 save", escapeHtml(legSummary(report, persistence.bm25))); + let rows = row("BM25 index", "rebuilt from stored content at boot"); if (persistence.vector) { rows += row("Vector save", escapeHtml(legSummary(report, persistence.vector))); - const chars = persistence.vector.serializedChars; - if (chars !== null) { - rows += row( - "Vector index size", - escapeHtml(`${chars.toLocaleString("en-US")} characters (${Math.round((chars / V8_MAX_STRING_CHARS) * 100)}% of the Node.js string limit)`), - ); - } + rows += row( + "Vector storage", + escapeHtml(`${persistence.buckets} buckets, ${persistence.pendingChanges} unsaved changes`), + ); } return rows; } @@ -396,6 +406,8 @@ ${row("Vector documents", escapeHtml(idx.vectorDocuments ?? "vector search off") ${row("Observations indexed", escapeHtml(idx.observationsIndexed))} ${row("Missing from index", escapeHtml(idx.missingObservations ?? "not checked"))} ${row("Sessions", escapeHtml(idx.sessions ?? "unknown"))} +${row("BM25 rebuild", idx.bm25Incomplete ? 'incomplete' : "complete")} +${row("Pending vector backfill", escapeHtml(idx.pendingVectorBackfill))} ${indexPersistenceRows(report)}

Knowledge graph

diff --git a/src/index.ts b/src/index.ts index ff465afe46..c043a31fab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,6 @@ import { createImageEmbeddingProvider, } from "./providers/index.js"; import { StateKV } from "./state/kv.js"; -import { KV } from "./state/schema.js"; import { VectorIndex } from "./state/vector-index.js"; import { HybridSearch } from "./state/hybrid-search.js"; import { IndexPersistence } from "./state/index-persistence.js"; @@ -35,13 +34,14 @@ import { registerDiskSizeManager } from "./functions/disk-size-manager.js"; import { registerCompressFunction } from "./functions/compress.js"; import { registerSearchFunction, - rebuildIndex, - reconcileIndex, + backfillVectors, + rebuildKeywordIndex, getSearchIndex, setVectorIndex, setEmbeddingProvider, setIndexPersistence, setHybridRanker, + setPendingVectorBackfillCount, } from "./functions/search.js"; import { registerContextFunction } from "./functions/context.js"; import { registerSummarizeFunction } from "./functions/summarize.js"; @@ -395,23 +395,13 @@ async function main() { const healthMonitor = registerHealthMonitor(sdk, kv); - const indexPersistence = new IndexPersistence(kv, bm25Index, vectorIndex); - // Wire the persistence hook so delete paths can flush BM25/vector - // index mutations to disk. Without this, an in-memory remove can be - // lost across a hard process exit and the persisted snapshot - // restores the deleted entry at next boot. + const indexPersistence = new IndexPersistence(kv, vectorIndex); setIndexPersistence(indexPersistence); const loaded = await indexPersistence.load().catch((err) => { - console.warn(`[agentmemory] Failed to load persisted index:`, err); + console.warn(`[agentmemory] Failed to load persisted vector index:`, err); return null; }); - if (loaded?.bm25 && loaded.bm25.size > 0) { - bm25Index.restoreFrom(loaded.bm25); - bootLog( - `Loaded persisted BM25 index (${bm25Index.size} docs)`, - ); - } if (loaded?.vector && vectorIndex && loaded.vector.size > 0) { // Persisted vectors carry whatever dimension the provider had when // they were written. If the active provider declares a different @@ -436,6 +426,7 @@ async function main() { const distinct = Array.from(seenDimensions).sort((a, b) => a - b).join(", "); const dropStale = isDropStaleIndexEnabled(); if (dropStale) { + for (const [obsId] of loaded.vector.entries()) vectorIndex.markRemoved(obsId); console.warn( `[agentmemory] Persisted vector index has ${mismatches.length} of ` + `${loaded.vector.size} vectors with the wrong dimension. Active ` + @@ -466,79 +457,42 @@ async function main() { } } - const needsRebuild = bm25Index.size === 0; - - if (needsRebuild) { - // Fire-and-forget. rebuildIndex iterates every observation across - // every session and AWAITS an embedding-provider call per record. - // On a large corpus + rate-limited embedding endpoint that can - // take HOURS; awaiting it here blocks every subsequent boot step - // (including startViewerServer below, leaving the viewer port - // unbound for the duration). The index lazily fills in over time - // and search degrades gracefully — partial coverage > no viewer - // for hours. Errors still surface via the inner .catch. - void rebuildIndex(kv) - .then((indexCount) => { - if (indexCount > 0) { - bootLog(`Search index rebuilt: ${indexCount} entries`); - indexPersistence.scheduleSave(); - } - }) - .catch((err) => { - console.warn(`[agentmemory] Failed to rebuild search index:`, err); - }); - } else { - // Backfill memories into BM25 for users upgrading from <0.9.5: prior - // versions of mem::remember never indexed memories, so the persisted - // BM25 covers observations only and `memory_smart_search` returns - // empty for everything saved via memory_save (#257). Walk KV.memories - // and add the ones missing from the restored index. Idempotent on - // re-runs because SearchIndex.has() short-circuits already-indexed - // ids. - try { - const memories = await kv.list(KV.memories); - let backfilled = 0; - for (const memory of memories) { - if (memory.isLatest === false) continue; - if (!memory.title || !memory.content) continue; - if (bm25Index.has(memory.id)) continue; - bm25Index.add({ - id: memory.id, - sessionId: memory.sessionIds?.[0] ?? "memory", - timestamp: memory.createdAt, - type: "decision", - title: memory.title, - facts: [memory.content], - narrative: memory.content, - concepts: memory.concepts, - files: memory.files, - importance: memory.strength, - }); - backfilled++; - } - if (backfilled > 0) { - bootLog( - `Backfilled ${backfilled} memories into BM25 (legacy index gap)`, - ); - indexPersistence.scheduleSave(); - } - } catch (err) { - console.warn( - `[agentmemory] Failed to backfill memories into BM25:`, - err, + const vectorCountShortfall = + Boolean(loaded?.vector) && + loaded?.expectedCount !== undefined && + loaded.vector!.size < loaded.expectedCount; + const vectorBackfillSince = + !loaded || loaded.state === "unavailable" + ? undefined + : loaded.state === "none" || vectorCountShortfall + ? null + : loaded.savedAt; + const keywordStart = Date.now(); + try { + const keyword = await rebuildKeywordIndex(kv, vectorBackfillSince); + bootLog( + `Rebuilt BM25 index from stored content (${keyword.documents} docs in ${Date.now() - keywordStart} ms)`, + ); + setPendingVectorBackfillCount(keyword.vectorJobs.length + keyword.fullBackfillPending); + if (keyword.fullBackfillPending > 0) { + bootLog( + `Vector backfill needs ${keyword.fullBackfillPending} embeddings but a full backfill was not started ` + + `(set AGENTMEMORY_VECTOR_BACKFILL=all to opt in). See /agentmemory/status.`, ); } - void reconcileIndex(kv) - .then((count) => { - if (count > 0) { - bootLog( - `Search index reconciled: ${count} observations missing from the persisted snapshot were re-indexed`, - ); - } - }) - .catch((err) => { - console.warn(`[agentmemory] Failed to reconcile search index:`, err); - }); + if (keyword.vectorJobs.length > 0) { + bootLog(`Backfilling ${keyword.vectorJobs.length} missing vectors in the background`); + void backfillVectors(keyword.vectorJobs) + .then((count) => { + setPendingVectorBackfillCount(keyword.fullBackfillPending); + if (count > 0) bootLog(`Vector index backfilled: ${count} entries`); + }) + .catch((err) => { + console.warn(`[agentmemory] Failed to backfill vectors:`, err); + }); + } + } catch (err) { + console.warn(`[agentmemory] Failed to rebuild the BM25 index:`, err); } // Ready / Endpoints lines are emitted via `bootLog` so they're diff --git a/src/state/index-persistence.ts b/src/state/index-persistence.ts index 28a24d5a1c..d046d0dd0f 100644 --- a/src/state/index-persistence.ts +++ b/src/state/index-persistence.ts @@ -1,30 +1,21 @@ -import { SearchIndex } from "./search-index.js"; -import { VectorIndex } from "./vector-index.js"; +import { VectorIndex, base64ToFloat32, float32ToBase64 } from "./vector-index.js"; import type { StateKV } from "./kv.js"; -import { KV, generateId } from "./schema.js"; +import { KV } from "./schema.js"; import { logger } from "../logger.js"; import { safeAudit } from "../functions/audit.js"; -import { getIndexSaveIntervalMs } from "../config.js"; +import { getIndexSaveIntervalMs, getVectorBucketSize } from "../config.js"; -export const V8_MAX_STRING_CHARS = 536_870_888; const FAILURE_LOG_THROTTLE_MS = 60_000; const INDEX_PERSISTENCE_FUNCTION_ID = "mem::index-persistence"; -const BM25_KEY = "data"; -const BM25_MANIFEST_KEY = "data:manifest"; -const BM25_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:bm25:`; -const VECTOR_KEY = "vectors"; -const VECTOR_MANIFEST_KEY = "vectors:manifest"; -const VECTOR_SHARD_SCOPE_PREFIX = `${KV.bm25Index}:vectors:`; -const INDEX_SHARD_KEY = "data"; -const DEFAULT_INDEX_SHARD_CHARS = 2_000_000; - -// mem:audit exists to record structural deletions of user data — that is -// the policy stated at the top of src/functions/audit.ts. Index shard -// writes and manifest publishes remove no user rows, so they fall outside -// it, yet a single save() emits three of them: on a real store they -// reached 59876 of 84028 entries (71%), which is what makes the audit log -// slow to query and bloats startup. Off by default; set -// AGENTMEMORY_AUDIT_INDEX_PERSIST=1 when debugging index persistence. +const LEGACY_BM25_KEY = "data"; +const LEGACY_BM25_MANIFEST_KEY = "data:manifest"; +const LEGACY_VECTOR_KEY = "vectors"; +const LEGACY_VECTOR_MANIFEST_KEY = "vectors:manifest"; +const VECTOR_META_KEY = "vectors:meta"; +const VECTOR_BUCKET_SCOPE_PREFIX = `${KV.bm25Index}:vec:`; +const WRITE_CONCURRENCY = 32; +const LOAD_CONCURRENCY = 8; + function auditIndexPersistEnabled(): boolean { const raw = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; if (!raw) return false; @@ -39,45 +30,57 @@ type IndexShardManifest = { chars: number; }; +type VectorMeta = { + v: 3; + bucketCount: number; + savedAt: string; + count: number; +}; + +type PersistedVector = { + id: string; + s: string; + e: string; +}; + type IndexPersistenceOptions = { - shardChars?: number; - createGeneration?: () => string; saveIntervalMs?: number; + bucketSize?: number; now?: () => number; }; -export type IndexLeg = "bm25" | "vector"; - export interface IndexLegStatus { lastSavedAt: string | null; lastError: string | null; lastErrorAt: string | null; dirtySince: string | null; - serializedChars: number | null; +} + +export interface VectorCountShortfall { + expected: number; + loaded: number; } export interface IndexPersistenceStatus { saveIntervalMs: number; saving: boolean; - bm25: IndexLegStatus; + buckets: number; + pendingChanges: number; vector: IndexLegStatus | null; + vectorCountShortfall: VectorCountShortfall | null; } -function emptyLegStatus(): IndexLegStatus { - return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null, serializedChars: null }; -} +export type VectorLoadState = "buckets" | "migrated" | "none" | "unavailable"; -function shardChars(options: IndexPersistenceOptions): number { - const configured = options.shardChars; - if (typeof configured !== "number" || !Number.isFinite(configured)) { - return DEFAULT_INDEX_SHARD_CHARS; - } - const wholeChars = Math.floor(configured); - return wholeChars >= 1 ? wholeChars : DEFAULT_INDEX_SHARD_CHARS; +export interface VectorLoadResult { + vector: VectorIndex | null; + state: VectorLoadState; + savedAt: string | null; + expectedCount?: number; } -function createIndexGeneration(): string { - return generateId("idx"); +export function vectorBucketScope(bucket: number): string { + return `${VECTOR_BUCKET_SCOPE_PREFIX}${String(bucket).padStart(4, "0")}`; } function statePath(scope: string, key: string): string { @@ -88,6 +91,13 @@ function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } +function generationTime(generation: string | undefined): string | null { + const stamp = generation?.split("_")[1]; + if (!stamp) return null; + const ms = parseInt(stamp, 36); + return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : null; +} + function isValidShardDescriptor( shard: unknown, ): shard is IndexShardManifest["shards"][number] { @@ -99,31 +109,55 @@ function isValidShardDescriptor( typeof candidate.key === "string" && candidate.key.length > 0 && Number.isInteger(candidate.chars) && - candidate.chars >= 0 + (candidate.chars as number) >= 0 ); } +function isPersistedVector(row: unknown): row is PersistedVector { + if (!row || typeof row !== "object") return false; + const candidate = row as Partial; + return typeof candidate.id === "string" && typeof candidate.s === "string" && typeof candidate.e === "string"; +} + +async function inBatches(items: T[], size: number, run: (item: T) => Promise): Promise { + const failures: unknown[] = []; + for (let offset = 0; offset < items.length; offset += size) { + const results = await Promise.allSettled(items.slice(offset, offset + size).map(run)); + for (const result of results) { + if (result.status === "rejected") failures.push(result.reason); + } + } + return failures; +} + +function emptyLegStatus(): IndexLegStatus { + return { lastSavedAt: null, lastError: null, lastErrorAt: null, dirtySince: null }; +} + export class IndexPersistence { private timer: ReturnType | null = null; - private lastFailureLogAt = new Map(); + private lastFailureLogAt = 0; private running: Promise | null = null; private queued: Promise | null = null; private stopped = false; private lastSaveAt: number; private dirtyEpoch = 0; private markedDuringRunAt: number | null = null; - private legs: Record = { - bm25: emptyLegStatus(), - vector: emptyLegStatus(), - }; + private metaWritten = false; + private leg: IndexLegStatus = emptyLegStatus(); private readonly saveIntervalMs: number; + private readonly bucketSize: number; private readonly now: () => number; + private bucketOfId: Map = new Map(); + private bucketCounts: Map = new Map(); + private highestBucket = 0; + private hasOpenBucket = false; + private vectorCountShortfall: VectorCountShortfall | null = null; constructor( private kv: StateKV, - private bm25: SearchIndex, private vector: VectorIndex | null, - private options: IndexPersistenceOptions = {}, + options: IndexPersistenceOptions = {}, ) { this.now = options.now ?? Date.now; const interval = options.saveIntervalMs; @@ -131,22 +165,25 @@ export class IndexPersistence { typeof interval === "number" && Number.isFinite(interval) && interval > 0 ? interval : getIndexSaveIntervalMs(); + const bucketSize = options.bucketSize; + this.bucketSize = + typeof bucketSize === "number" && Number.isInteger(bucketSize) && bucketSize > 0 + ? bucketSize + : getVectorBucketSize(); this.lastSaveAt = this.now(); } scheduleSave(): void { - if (this.stopped) return; + if (this.stopped || !this.vector) return; const now = this.now(); this.dirtyEpoch++; if (this.running && this.markedDuringRunAt === null) this.markedDuringRunAt = now; - for (const leg of this.activeLegs()) { - if (this.legs[leg].dirtySince === null) this.legs[leg].dirtySince = new Date(now).toISOString(); - } + if (this.leg.dirtySince === null) this.leg.dirtySince = new Date(now).toISOString(); if (this.timer) return; const delay = Math.max(0, this.lastSaveAt + this.saveIntervalMs - now); this.timer = setTimeout(() => { this.timer = null; - this.save().catch((err) => this.logFailure("bm25", err)); + this.save().catch((err) => this.logFailure(err)); }, delay); } @@ -168,34 +205,79 @@ export class IndexPersistence { return { saveIntervalMs: this.saveIntervalMs, saving: this.running !== null, - bm25: { ...this.legs.bm25 }, - vector: this.vector ? { ...this.legs.vector } : null, + buckets: this.bucketCountInUse(), + pendingChanges: this.vector?.pendingChanges ?? 0, + vector: this.vector ? { ...this.leg } : null, + vectorCountShortfall: this.vectorCountShortfall, }; } - async load(): Promise<{ - bm25: SearchIndex | null; - vector: VectorIndex | null; - }> { - let bm25: SearchIndex | null = null; - let vector: VectorIndex | null = null; + stop(): void { + this.stopped = true; + this.clearTimer(); + } + + async load(): Promise { + if (!this.vector) { + await this.removeLegacyBm25Snapshot(); + return { vector: null, state: "none", savedAt: null }; + } - const bm25Data = await this.loadBm25Data(); - if (bm25Data && typeof bm25Data === "string") { - bm25 = SearchIndex.deserialize(bm25Data); + let meta: VectorMeta | null; + try { + meta = await this.kv.get(KV.bm25Index, VECTOR_META_KEY); + } catch (err) { + logger.warn("index persistence: vector metadata read failed", { message: errorMessage(err) }); + return { vector: null, state: "unavailable", savedAt: null }; } - const vecData = await this.loadVectorData(); - if (vecData && typeof vecData === "string") { - vector = VectorIndex.deserialize(vecData); + if (meta && meta.v === 3 && Number.isInteger(meta.bucketCount) && meta.bucketCount >= 0) { + const loaded = await this.loadBuckets(meta.bucketCount); + if (!loaded) return { vector: null, state: "unavailable", savedAt: null }; + this.metaWritten = true; + await this.removeLegacyVectorSnapshotIfPresent(); + await this.removeLegacyBm25Snapshot(); + const expectedCount = Number.isInteger(meta.count) ? meta.count : undefined; + this.vectorCountShortfall = + expectedCount !== undefined && loaded.size < expectedCount + ? { expected: expectedCount, loaded: loaded.size } + : null; + if (this.vectorCountShortfall) { + logger.warn("index persistence: loaded fewer vectors than the last save recorded", { + ...this.vectorCountShortfall, + }); + } + return { vector: loaded, state: "buckets", savedAt: meta.savedAt ?? null, expectedCount }; } - return { bm25, vector }; + return this.migrateLegacy(); } - stop(): void { - this.stopped = true; - this.clearTimer(); + private bucketCountInUse(): number { + return this.hasOpenBucket ? this.highestBucket + 1 : 0; + } + + private assignBucket(id: string): number { + const existing = this.bucketOfId.get(id); + if (existing !== undefined) return existing; + if (!this.hasOpenBucket) { + this.hasOpenBucket = true; + this.highestBucket = 0; + } else if ((this.bucketCounts.get(this.highestBucket) ?? 0) >= this.bucketSize) { + this.highestBucket++; + } + this.bucketOfId.set(id, this.highestBucket); + this.bucketCounts.set(this.highestBucket, (this.bucketCounts.get(this.highestBucket) ?? 0) + 1); + return this.highestBucket; + } + + private releaseBucket(id: string): void { + const bucket = this.bucketOfId.get(id); + if (bucket === undefined) return; + this.bucketOfId.delete(id); + const remaining = (this.bucketCounts.get(bucket) ?? 1) - 1; + if (remaining > 0) this.bucketCounts.set(bucket, remaining); + else this.bucketCounts.delete(bucket); } private clearTimer(): void { @@ -205,10 +287,6 @@ export class IndexPersistence { } } - private activeLegs(): IndexLeg[] { - return this.vector ? ["bm25", "vector"] : ["bm25"]; - } - private startRun(): Promise { const run = this.runSave().finally(() => { if (this.running === run) this.running = null; @@ -218,176 +296,199 @@ export class IndexPersistence { } private async runSave(): Promise { - this.markedDuringRunAt = null; - this.lastSaveAt = this.now(); - await this.saveLeg("bm25", async () => { - const serialized = this.bm25.serialize(); - this.legs.bm25.serializedChars = serialized.length; - await this.saveBm25Index(serialized); - }); const vector = this.vector; - if (vector) { - await this.saveLeg("vector", async () => { - const serialized = vector.serialize(); - this.legs.vector.serializedChars = serialized.length; - await this.saveVectorIndex(serialized); - }); - } - } - - private async saveLeg(leg: IndexLeg, run: () => Promise): Promise { - const status = this.legs[leg]; + if (!vector) return; const epoch = this.dirtyEpoch; + this.markedDuringRunAt = null; + this.lastSaveAt = this.now(); try { - await run(); - status.lastSavedAt = new Date(this.now()).toISOString(); - status.lastError = null; - status.lastErrorAt = null; - if (this.dirtyEpoch === epoch) { - status.dirtySince = null; + await this.writeChanges(vector); + this.leg.lastSavedAt = new Date(this.now()).toISOString(); + this.leg.lastError = null; + this.leg.lastErrorAt = null; + if (this.dirtyEpoch === epoch || vector.pendingChanges === 0) { + this.leg.dirtySince = null; } else if (this.markedDuringRunAt !== null) { - status.dirtySince = new Date(this.markedDuringRunAt).toISOString(); + this.leg.dirtySince = new Date(this.markedDuringRunAt).toISOString(); } } catch (err) { - status.lastError = errorMessage(err); - status.lastErrorAt = new Date(this.now()).toISOString(); - if (status.dirtySince === null) status.dirtySince = status.lastErrorAt; - this.logFailure(leg, err); + this.leg.lastError = errorMessage(err); + this.leg.lastErrorAt = new Date(this.now()).toISOString(); + if (this.leg.dirtySince === null) this.leg.dirtySince = this.leg.lastErrorAt; + this.logFailure(err); } } - private logFailure(leg: IndexLeg, err: unknown): void { - const now = this.now(); - if (now - (this.lastFailureLogAt.get(leg) ?? 0) < FAILURE_LOG_THROTTLE_MS) return; - this.lastFailureLogAt.set(leg, now); - const code = (err as { code?: string })?.code; - const message = err instanceof Error ? err.message : String(err); - logger.warn(`index persistence: failed to save the ${leg === "bm25" ? "BM25" : "vector"} index`, { - code, - message, - hint: - code === "TIMEOUT" - ? "iii-engine state::set timed out; recent index updates remain in memory and will retry on the next save" - : undefined, + private async writeChanges(vector: VectorIndex): Promise { + const changes = vector.takeChanges(); + if (changes.size === 0 && this.metaWritten) return; + const failed = new Map(); + const failures = await inBatches([...changes], WRITE_CONCURRENCY, async ([id, present]) => { + const entry = present ? vector.get(id) : undefined; + try { + if (entry) { + const bucket = this.assignBucket(id); + await this.kv.set(vectorBucketScope(bucket), id, { + id, + s: entry.sessionId, + e: float32ToBase64(entry.embedding), + }); + } else { + const bucket = this.bucketOfId.get(id); + if (bucket !== undefined) { + await this.kv.delete(vectorBucketScope(bucket), id); + this.releaseBucket(id); + } + } + } catch (err) { + failed.set(id, present); + throw err; + } }); + if (failures.length > 0) { + vector.returnChanges(failed); + throw new Error( + `${failures.length} of ${changes.size} vector writes failed: ${errorMessage(failures[0])}`, + ); + } + await this.kv.set(KV.bm25Index, VECTOR_META_KEY, { + v: 3, + bucketCount: this.bucketCountInUse(), + savedAt: new Date(this.now()).toISOString(), + count: vector.size, + }); + this.metaWritten = true; } - private async saveBm25Index(serialized: string): Promise { - await this.saveShardedIndex( - serialized, - BM25_MANIFEST_KEY, - BM25_KEY, - BM25_SHARD_SCOPE_PREFIX, - ); + private async loadBuckets(bucketCount: number): Promise { + const loaded = new VectorIndex(); + const bucketOfId = new Map(); + const bucketCounts = new Map(); + const bucketIds = Array.from({ length: bucketCount }, (_, bucket) => bucket); + const failures = await inBatches(bucketIds, LOAD_CONCURRENCY, async (bucket) => { + const rows = await this.kv.list(vectorBucketScope(bucket)); + let count = 0; + for (const row of rows) { + if (!isPersistedVector(row)) continue; + try { + loaded.loadPersisted(row.id, row.s, base64ToFloat32(row.e)); + } catch { + continue; + } + bucketOfId.set(row.id, bucket); + count++; + } + if (count > 0) bucketCounts.set(bucket, count); + }); + if (failures.length > 0) { + logger.warn("index persistence: vector bucket read failed", { + failed: failures.length, + message: errorMessage(failures[0]), + }); + return null; + } + this.bucketOfId = bucketOfId; + this.bucketCounts = bucketCounts; + this.highestBucket = bucketCount > 0 ? bucketCount - 1 : 0; + this.hasOpenBucket = bucketCount > 0; + return loaded; } - private async saveVectorIndex(serialized: string): Promise { - await this.saveShardedIndex( - serialized, - VECTOR_MANIFEST_KEY, - VECTOR_KEY, - VECTOR_SHARD_SCOPE_PREFIX, + private async migrateLegacy(): Promise { + const manifest = await this.readIndexValue( + KV.bm25Index, + LEGACY_VECTOR_MANIFEST_KEY, + "vector", + "manifest", ); - } - - private async saveShardedIndex( - serialized: string, - manifestKey: string, - legacyKey: string, - scopePrefix: string, - ): Promise { - const previous = await this.kv - .get(KV.bm25Index, manifestKey) - .catch(() => null); - const generation = - this.options.createGeneration?.() ?? createIndexGeneration(); - const chunkChars = shardChars(this.options); - const shards: IndexShardManifest["shards"] = []; - const chunks: string[] = []; - - for (let offset = 0; offset < serialized.length; offset += chunkChars) { - const shardIndex = shards.length; - const scope = `${scopePrefix}${generation}:${String(shardIndex).padStart( - 5, - "0", - )}`; - const chunk = serialized.slice(offset, offset + chunkChars); - shards.push({ scope, key: INDEX_SHARD_KEY, chars: chunk.length }); - chunks.push(chunk); + const legacyKey = await this.readIndexValue(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy"); + if (!manifest.ok || !legacyKey.ok) return { vector: null, state: "unavailable", savedAt: null }; + const legacyPresent = manifest.value != null || (typeof legacyKey.value === "string" && legacyKey.value.length > 0); + if (!legacyPresent) { + await this.removeLegacyBm25Snapshot(); + return { vector: null, state: "none", savedAt: null }; } - const writeResults = await Promise.allSettled( - shards.map(async (shard, index) => { - const chunk = chunks[index] ?? ""; - await this.kv.set(shard.scope, shard.key, chunk); - await this.auditIndexPersistence("shard_write", [ - statePath(shard.scope, shard.key), - ], { - scope: shard.scope, - key: shard.key, - manifestKey, - generation, - chars: chunk.length, - }); - }), - ); - const failedWrite = writeResults.find( - (result): result is PromiseRejectedResult => result.status === "rejected", - ); - if (failedWrite) { - await this.deleteShards(shards, "shard_write_rollback"); - throw failedWrite.reason; + const data = await this.loadShardedData(LEGACY_VECTOR_KEY, LEGACY_VECTOR_MANIFEST_KEY, "vector"); + if (typeof data !== "string") { + logger.warn("index persistence: legacy vector snapshot is unreadable; keeping it and skipping migration"); + return { vector: null, state: "unavailable", savedAt: null }; } + const legacy = VectorIndex.deserialize(data); + const savedAt = + (manifest.value && typeof manifest.value === "object" ? generationTime(manifest.value.generation) : null) ?? + new Date(this.now()).toISOString(); - const nextManifest: IndexShardManifest = { - v: 1, - generation, - shards, - chars: serialized.length, - }; + const staging = new VectorIndex(); + for (const [id, entry] of legacy.entries()) staging.add(id, entry.sessionId, entry.embedding); try { - await this.kv.set( - KV.bm25Index, - manifestKey, - nextManifest, - ); - await this.auditIndexPersistence("manifest_publish", [ - statePath(KV.bm25Index, manifestKey), - ], { - manifestKey, - generation, - chars: serialized.length, - shards: shards.length, - result: "committed", - }); + await this.writeChanges(staging); } catch (err) { - if (await this.isManifestPublished(manifestKey, nextManifest)) { - await this.auditIndexPersistence("manifest_publish", [ - statePath(KV.bm25Index, manifestKey), - ], { - manifestKey, - generation, - chars: serialized.length, - shards: shards.length, - result: "committed_after_error", - error: errorMessage(err), - }); - } else { - await this.deleteShards(shards, "manifest_publish_rollback"); - } - throw err; + logger.warn("index persistence: migrating the legacy vector snapshot failed; it stays in place and the next save retries", { + message: errorMessage(err), + }); + legacy.markAllChanged(); + return { vector: legacy, state: "unavailable", savedAt }; } - await this.deleteKey(KV.bm25Index, legacyKey, "legacy_cleanup"); - if (previous?.v === 1 && Array.isArray(previous.shards)) { - const currentShardIds = new Set( - shards.map((shard) => `${shard.scope}\0${shard.key}`), - ); - for (const shard of previous.shards) { - if (currentShardIds.has(`${shard.scope}\0${shard.key}`)) continue; - await this.deleteShards([shard], "previous_generation_cleanup"); - } + await this.removeLegacyVectorSnapshot(manifest.value); + await this.removeLegacyBm25Snapshot(); + await this.auditIndexPersistence("migrate", [statePath(KV.bm25Index, VECTOR_META_KEY)], { + vectors: legacy.size, + buckets: this.bucketCountInUse(), + }); + logger.info("index persistence: migrated the vector index to bucketed storage", { + vectors: legacy.size, + buckets: this.bucketCountInUse(), + }); + const migrated = new VectorIndex(); + for (const [id, entry] of legacy.entries()) migrated.loadPersisted(id, entry.sessionId, entry.embedding); + return { vector: migrated, state: "migrated", savedAt }; + } + + private async removeLegacyVectorSnapshot(manifest: IndexShardManifest | null): Promise { + if (manifest && Array.isArray(manifest.shards)) { + await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_vector_cleanup"); + } + await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "legacy_vector_cleanup"); + await this.deleteKey(KV.bm25Index, LEGACY_VECTOR_KEY, "legacy_vector_cleanup"); + } + + private async removeLegacyVectorSnapshotIfPresent(): Promise { + const manifest = await this.readIndexValue(KV.bm25Index, LEGACY_VECTOR_MANIFEST_KEY, "vector", "manifest"); + const legacy = await this.readIndexValue(KV.bm25Index, LEGACY_VECTOR_KEY, "vector", "legacy"); + if (!manifest.ok || !legacy.ok) return; + if (manifest.value == null && legacy.value == null) return; + await this.removeLegacyVectorSnapshot(manifest.value ?? null); + } + + private async removeLegacyBm25Snapshot(): Promise { + const manifestRead = await this.readIndexValue(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "bm25", "manifest"); + const legacyRead = await this.readIndexValue(KV.bm25Index, LEGACY_BM25_KEY, "bm25", "legacy"); + if (!manifestRead.ok || !legacyRead.ok) return; + const manifest = manifestRead.value ?? null; + const legacy = legacyRead.value ?? null; + if (manifest == null && legacy == null) return; + if (manifest && Array.isArray(manifest.shards)) { + await this.deleteShards(manifest.shards.filter(isValidShardDescriptor), "legacy_bm25_cleanup"); } + if (manifest != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_MANIFEST_KEY, "legacy_bm25_cleanup"); + if (legacy != null) await this.deleteKey(KV.bm25Index, LEGACY_BM25_KEY, "legacy_bm25_cleanup"); + } + + private logFailure(err: unknown): void { + const now = this.now(); + if (now - this.lastFailureLogAt < FAILURE_LOG_THROTTLE_MS) return; + this.lastFailureLogAt = now; + const code = (err as { code?: string })?.code; + logger.warn("index persistence: failed to save the vector index", { + code, + message: errorMessage(err), + hint: + code === "TIMEOUT" + ? "iii-engine state::set timed out; unsaved vectors stay in memory and retry on the next save" + : undefined, + }); } private async auditIndexPersistence( @@ -396,20 +497,13 @@ export class IndexPersistence { details: Record, ): Promise { if (!auditIndexPersistEnabled()) return; - await safeAudit( - this.kv, - "index_persist", - INDEX_PERSISTENCE_FUNCTION_ID, - targetIds, - { action, ...details }, - ); + await safeAudit(this.kv, "index_persist", INDEX_PERSISTENCE_FUNCTION_ID, targetIds, { + action, + ...details, + }); } - private async deleteKey( - scope: string, - key: string, - reason: string, - ): Promise { + private async deleteKey(scope: string, key: string, reason: string): Promise { let result = "deleted"; let error: string | undefined; try { @@ -427,81 +521,17 @@ export class IndexPersistence { }); } - private async deleteShards( - shards: IndexShardManifest["shards"], - reason: string, - ): Promise { - for (const shard of shards) { - await this.deleteKey(shard.scope, shard.key, reason); - } - } - - private async isManifestPublished( - manifestKey: string, - expected: IndexShardManifest, - ): Promise { - const published = await this.kv - .get(KV.bm25Index, manifestKey) - .catch(() => null); - if ( - published?.v !== 1 || - published.generation !== expected.generation || - published.chars !== expected.chars || - !Array.isArray(published.shards) || - published.shards.length !== expected.shards.length - ) { - return false; - } - return published.shards.every((shard, index) => { - const expectedShard = expected.shards[index]; - if (!expectedShard) return false; - return ( - shard.scope === expectedShard.scope && - shard.key === expectedShard.key && - shard.chars === expectedShard.chars - ); - }); - } - - private async loadBm25Data(): Promise { - return this.loadShardedData(BM25_KEY, BM25_MANIFEST_KEY, "BM25"); + private async deleteShards(shards: IndexShardManifest["shards"], reason: string): Promise { + await inBatches(shards, WRITE_CONCURRENCY, (shard) => this.deleteKey(shard.scope, shard.key, reason)); } - private async loadVectorData(): Promise { - return this.loadShardedData(VECTOR_KEY, VECTOR_MANIFEST_KEY, "vector"); - } - - private async loadShardedData( - legacyKey: string, - manifestKey: string, - label: string, - ): Promise { - const manifest = await this.readIndexValue( - KV.bm25Index, - manifestKey, - label, - "manifest", - ); + private async loadShardedData(legacyKey: string, manifestKey: string, label: string): Promise { + const manifest = await this.readIndexValue(KV.bm25Index, manifestKey, label, "manifest"); if (!manifest.ok) return null; - // #797: some iii-state adapters return `undefined` (not `null`) for - // a missing key. The previous `value !== null` check passed - // undefined through to loadManifestData, which then crashed on - // `manifest.v` with TypeError. Treat both null and undefined as - // "no manifest" and fall through to the legacy path. The shape - // check stays so a malformed-but-present row still fails closed. - if ( - manifest.value != null && - typeof manifest.value === "object" - ) { + if (manifest.value != null && typeof manifest.value === "object") { return this.loadManifestData(manifest.value, label); } - - const legacy = await this.readIndexValue( - KV.bm25Index, - legacyKey, - label, - "legacy", - ); + const legacy = await this.readIndexValue(KV.bm25Index, legacyKey, label, "legacy"); if (!legacy.ok) return null; if (legacy.value && typeof legacy.value === "string") return legacy.value; return null; @@ -525,10 +555,7 @@ export class IndexPersistence { } } - private async loadManifestData( - manifest: IndexShardManifest, - label: string, - ): Promise { + private async loadManifestData(manifest: IndexShardManifest, label: string): Promise { if ( manifest.v !== 1 || !Array.isArray(manifest.shards) || @@ -555,10 +582,7 @@ export class IndexPersistence { let chars = 0; for (const { shard, chunk } of loadedShards) { if (typeof chunk !== "string") { - logger.warn(`index persistence: ${label} shard missing`, { - scope: shard.scope, - key: shard.key, - }); + logger.warn(`index persistence: ${label} shard missing`, { scope: shard.scope, key: shard.key }); return null; } if (chunk.length !== shard.chars) { @@ -574,10 +598,7 @@ export class IndexPersistence { chars += chunk.length; } if (chars !== manifest.chars) { - logger.warn(`index persistence: ${label} total length mismatch`, { - expected: manifest.chars, - actual: chars, - }); + logger.warn(`index persistence: ${label} total length mismatch`, { expected: manifest.chars, actual: chars }); return null; } return chunks.join(""); diff --git a/src/state/vector-index.ts b/src/state/vector-index.ts index d4b8bda760..2425bef8e1 100644 --- a/src/state/vector-index.ts +++ b/src/state/vector-index.ts @@ -5,13 +5,13 @@ // pool. Same risk on the encode side if the input Float32Array is itself // a sliced view. Reported as a phantom "2048 dimensions on disk" crash // in #455 / #469 / #584 / #587. -function float32ToBase64(arr: Float32Array): string { +export function float32ToBase64(arr: Float32Array): string { return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).toString( "base64", ); } -function base64ToFloat32(b64: string): Float32Array { +export function base64ToFloat32(b64: string): Float32Array { const buf = Buffer.from(b64, "base64"); return new Float32Array( buf.buffer, @@ -34,16 +34,59 @@ function cosineSimilarity(a: Float32Array, b: Float32Array): number { return denom === 0 ? 0 : dot / denom; } +export type VectorEntry = { embedding: Float32Array; sessionId: string }; + export class VectorIndex { - private vectors: Map = - new Map(); + private vectors: Map = new Map(); + private changes: Map = new Map(); add(obsId: string, sessionId: string, embedding: Float32Array): void { this.vectors.set(obsId, { embedding, sessionId }); + this.changes.set(obsId, true); } remove(obsId: string): void { - this.vectors.delete(obsId); + if (this.vectors.delete(obsId)) this.changes.set(obsId, false); + } + + has(obsId: string): boolean { + return this.vectors.has(obsId); + } + + get(obsId: string): VectorEntry | undefined { + return this.vectors.get(obsId); + } + + entries(): IterableIterator<[string, VectorEntry]> { + return this.vectors.entries(); + } + + loadPersisted(obsId: string, sessionId: string, embedding: Float32Array): void { + this.vectors.set(obsId, { embedding, sessionId }); + } + + get pendingChanges(): number { + return this.changes.size; + } + + takeChanges(): Map { + const taken = this.changes; + this.changes = new Map(); + return taken; + } + + returnChanges(changes: Map): void { + for (const [obsId, present] of changes) { + if (!this.changes.has(obsId)) this.changes.set(obsId, present); + } + } + + markRemoved(obsId: string): void { + if (!this.vectors.has(obsId)) this.changes.set(obsId, false); + } + + markAllChanged(): void { + for (const obsId of this.vectors.keys()) this.changes.set(obsId, true); } search( @@ -104,6 +147,7 @@ export class VectorIndex { } clear(): void { + for (const obsId of this.vectors.keys()) this.changes.set(obsId, false); this.vectors.clear(); } @@ -119,6 +163,7 @@ export class VectorIndex { sessionId: entry.sessionId, }); } + this.changes = new Map(other.changes); } serialize(): string { diff --git a/src/triggers/api.ts b/src/triggers/api.ts index 2fc2f3285a..337e34b5d5 100644 --- a/src/triggers/api.ts +++ b/src/triggers/api.ts @@ -12,7 +12,14 @@ import type { ResilientProvider } from "../providers/resilient.js"; import { III_PINNED_VERSION, VERSION } from "../version.js"; import { CONSOLIDATION_COUNTS_REUSE_MS, CONSOLIDATION_LAST_RUN_KEY, PROCEDURAL_MIN_SESSIONS_PER_PATTERN, describeConsolidation, type ConsolidationRunRecord } from "../functions/consolidation-status.js"; import { UNINDEXED_SCAN_REUSE_MS, evaluateStatus, prefersHtml, renderStatusHtml, singleFlight, type GraphStatsInput } from "../functions/status.js"; -import { findUnindexedObservations, getIndexPersistenceStatus, getSearchIndex, getVectorIndex } from "../functions/search.js"; +import { + findUnindexedObservations, + getIndexPersistenceStatus, + getPendingVectorBackfillCount, + getSearchIndex, + getVectorIndex, + isBm25RebuildIncomplete, +} from "../functions/search.js"; import { timingSafeCompare } from "../auth.js"; import { isSlotsEnabled, isReflectEnabled } from "../functions/slots.js"; import { renderViewerDocument } from "../viewer/document.js"; @@ -370,6 +377,8 @@ export function registerApiTriggers( observationsIndexed, missingObservations: unindexed ? unindexed.missing.length : null, sessions: unindexed ? unindexed.sessions : null, + bm25Incomplete: isBm25RebuildIncomplete(), + pendingVectorBackfill: getPendingVectorBackfillCount(), }, graph, graphExtractionEnabled: isGraphExtractionEnabled(), diff --git a/test/index-keyword-rebuild.test.ts b/test/index-keyword-rebuild.test.ts new file mode 100644 index 0000000000..f0080675ba --- /dev/null +++ b/test/index-keyword-rebuild.test.ts @@ -0,0 +1,240 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { + backfillVectors, + getPendingVectorBackfillCount, + getSearchIndex, + isBm25RebuildIncomplete, + isMemoryIndexReady, + rebuildKeywordIndex, + setEmbeddingProvider, + setIndexPersistence, + setVectorIndex, +} from "../src/functions/search.js"; +import { VectorIndex } from "../src/state/vector-index.js"; +import type { CompressedObservation, EmbeddingProvider, Memory, Session } from "../src/types.js"; + +function mockKV() { + const store = new Map>(); + return { + store, + list: async (scope: string): Promise => Array.from(store.get(scope)?.values() ?? []) as T[], + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + }; +} + +const WORDS = ["auth", "cache", "index", "vector", "session", "memory", "graph", "retry", "queue", "schema", "deploy", "worker"]; + +function makeObs(id: string, sessionId: string, i: number, timestamp: string): CompressedObservation { + const a = WORDS[i % WORDS.length]; + const b = WORDS[(i * 7) % WORDS.length]; + return { + id, + sessionId, + timestamp, + type: "file_edit", + title: `${a} change ${i}`, + facts: [`touched ${b} path`], + narrative: `Updated the ${a} module so ${b} handling works for case ${i}`, + concepts: [a, b], + files: [`src/${a}/${b}.ts`], + importance: 5, + }; +} + +function seed(kv: ReturnType, sessions: number, perSession: number, timestamp: (i: number) => string) { + let n = 0; + for (let s = 0; s < sessions; s++) { + const sessionId = `ses_${s}`; + kv.store.set("mem:sessions", kv.store.get("mem:sessions") ?? new Map()); + kv.store.get("mem:sessions")!.set(sessionId, { + id: sessionId, + project: "p", + cwd: "/tmp", + startedAt: timestamp(0), + status: "completed", + observationCount: perSession, + } satisfies Session); + const scope = `mem:obs:${sessionId}`; + kv.store.set(scope, new Map()); + for (let o = 0; o < perSession; o++) { + const id = `obs_${n}`; + kv.store.get(scope)!.set(id, makeObs(id, sessionId, n, timestamp(n))); + n++; + } + } + const memories = new Map(); + memories.set("mem_1", { + id: "mem_1", + createdAt: timestamp(0), + updatedAt: timestamp(0), + type: "fact", + title: "Deploy uses blue green", + content: "Deploys switch traffic after health checks pass", + concepts: ["deploy"], + files: [], + sessionIds: ["ses_0"], + strength: 7, + version: 1, + isLatest: true, + } as Memory); + memories.set("mem_old", { + id: "mem_old", + createdAt: timestamp(0), + updatedAt: timestamp(0), + type: "fact", + title: "Superseded", + content: "Old version", + concepts: [], + files: [], + sessionIds: [], + strength: 1, + version: 1, + isLatest: false, + } as Memory); + kv.store.set("mem:memories", memories); + kv.store.get("mem:obs:ses_0")!.set("raw_1", { id: "raw_1", sessionId: "ses_0", timestamp: timestamp(0) }); + return n; +} + +function stubProvider(): EmbeddingProvider { + return { + name: "stub", + dimensions: 3, + embed: async () => new Float32Array([0.1, 0.2, 0.3]), + embedBatch: async (texts: string[]) => texts.map(() => new Float32Array([0.1, 0.2, 0.3])), + } as EmbeddingProvider; +} + +describe("rebuildKeywordIndex", () => { + afterEach(() => { + getSearchIndex().clear(); + setVectorIndex(null); + setEmbeddingProvider(null); + setIndexPersistence(null); + }); + + it("rebuilds BM25 from stored observations and latest memories, quickly", async () => { + const kv = mockKV(); + const total = seed(kv, 100, 30, () => "2026-09-01T00:00:00.000Z"); + getSearchIndex().add(makeObs("ghost", "ses_gone", 0, "2026-09-01T00:00:00.000Z")); + + const started = performance.now(); + const result = await rebuildKeywordIndex(kv as never); + const elapsed = performance.now() - started; + + expect(total).toBe(3000); + expect(result.documents).toBe(3001); + expect(result.vectorJobs).toEqual([]); + const idx = getSearchIndex(); + expect(idx.size).toBe(3001); + expect(idx.has("ghost")).toBe(false); + expect(idx.has("mem_1")).toBe(true); + expect(idx.has("mem_old")).toBe(false); + expect(idx.has("raw_1")).toBe(false); + expect(idx.search("blue green").map((r) => r.obsId)).toContain("mem_1"); + expect(isMemoryIndexReady()).toBe(true); + expect(elapsed).toBeLessThan(10_000); + }); + + it("keeps the vector index untouched and reports an incomplete rebuild when the sessions listing fails", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + const vector = new VectorIndex(); + vector.add("obs_0", "ses_0", new Float32Array([0.1, 0.2, 0.3])); + setVectorIndex(vector); + setEmbeddingProvider(stubProvider()); + const failingKv = { + ...kv, + list: async (scope: string): Promise => { + if (scope === "mem:sessions") throw new Error("engine unreachable"); + return kv.list(scope); + }, + }; + + const result = await rebuildKeywordIndex(failingKv as never, null); + + expect(result.vectorJobs).toEqual([]); + expect(vector.size).toBe(1); + expect(vector.has("obs_0")).toBe(true); + expect(isBm25RebuildIncomplete()).toBe(true); + }, 10_000); + + it("queues embeddings only for documents newer than the last vector save", async () => { + const kv = mockKV(); + seed(kv, 2, 5, (i) => (i < 5 ? "2026-09-01T00:00:00.000Z" : "2026-09-20T00:00:00.000Z")); + const vector = new VectorIndex(); + vector.add("obs_6", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + setVectorIndex(vector); + setEmbeddingProvider(stubProvider()); + + const result = await rebuildKeywordIndex(kv as never, "2026-09-10T00:00:00.000Z"); + + expect(result.vectorJobs.map((j) => j.id).sort()).toEqual(["obs_5", "obs_7", "obs_8", "obs_9"]); + }); + + it("withholds a whole-store backfill by default and reports it as pending, and queues none when storage was unavailable", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + setVectorIndex(new VectorIndex()); + setEmbeddingProvider(stubProvider()); + + const fresh = await rebuildKeywordIndex(kv as never, null); + expect(fresh.vectorJobs).toEqual([]); + expect(fresh.fullBackfillPending).toBe(4); + + const unavailable = await rebuildKeywordIndex(kv as never, undefined); + expect(unavailable.vectorJobs).toEqual([]); + expect(unavailable.fullBackfillPending).toBe(0); + }); + + it("runs the whole-store backfill, capped, once AGENTMEMORY_VECTOR_BACKFILL=all is set", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + setVectorIndex(new VectorIndex()); + setEmbeddingProvider(stubProvider()); + process.env.AGENTMEMORY_VECTOR_BACKFILL = "all"; + try { + const opted = await rebuildKeywordIndex(kv as never, null); + expect(opted.vectorJobs.map((j) => j.id).sort()).toEqual(["mem_1", "obs_0", "obs_1", "obs_2"]); + expect(opted.fullBackfillPending).toBe(0); + + process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX = "2"; + const capped = await rebuildKeywordIndex(kv as never, null); + expect(capped.vectorJobs.length).toBe(2); + } finally { + delete process.env.AGENTMEMORY_VECTOR_BACKFILL; + delete process.env.AGENTMEMORY_VECTOR_BACKFILL_MAX; + } + }); + + it("backfills queued vectors in batches and saves progress", async () => { + const kv = mockKV(); + seed(kv, 1, 3, () => "2026-09-01T00:00:00.000Z"); + const vector = new VectorIndex(); + setVectorIndex(vector); + setEmbeddingProvider(stubProvider()); + let saved = 0; + setIndexPersistence({ scheduleSave: () => {}, save: async () => void saved++ }); + process.env.AGENTMEMORY_VECTOR_BACKFILL = "all"; + + try { + const { vectorJobs } = await rebuildKeywordIndex(kv as never, null); + const added = await backfillVectors(vectorJobs); + + expect(added).toBe(4); + expect(vector.size).toBe(4); + expect(saved).toBe(1); + expect(getPendingVectorBackfillCount()).toBe(0); + } finally { + delete process.env.AGENTMEMORY_VECTOR_BACKFILL; + } + }); +}); diff --git a/test/index-persistence-throttle.test.ts b/test/index-persistence-throttle.test.ts index 54eff133b8..bc03834d41 100644 --- a/test/index-persistence-throttle.test.ts +++ b/test/index-persistence-throttle.test.ts @@ -1,18 +1,15 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { IndexPersistence, V8_MAX_STRING_CHARS } from "../src/state/index-persistence.js"; -import { SearchIndex } from "../src/state/search-index.js"; +import { IndexPersistence } from "../src/state/index-persistence.js"; import { VectorIndex } from "../src/state/vector-index.js"; import { evaluateStatus, renderStatusHtml, type StatusInputs } from "../src/functions/status.js"; -import type { CompressedObservation } from "../src/types.js"; -const BM25_SCOPE = "mem:index:bm25"; +const META_KEY = "vectors:meta"; function mockKV() { const store = new Map>(); const sets: Array<{ scope: string; key: string }> = []; return { sets, - store, get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, set: async (scope: string, key: string, data: T): Promise => { sets.push({ scope, key }); @@ -27,23 +24,12 @@ function mockKV() { }; } -function obs(id: string, title: string): CompressedObservation { - return { - id, - sessionId: "ses_1", - timestamp: new Date().toISOString(), - type: "file_edit", - title, - facts: [], - narrative: `${title} narrative`, - concepts: [], - files: [], - importance: 5, - }; +function metaSaves(kv: ReturnType): number { + return kv.sets.filter((s) => s.key === META_KEY).length; } -function manifestSaves(kv: ReturnType, key: string): number { - return kv.sets.filter((s) => s.scope === BM25_SCOPE && s.key === key).length; +function touch(vector: VectorIndex, id: string): void { + vector.add(id, "ses_1", new Float32Array([Math.random(), 0.2, 0.3])); } describe("IndexPersistence save throttling", () => { @@ -59,35 +45,40 @@ describe("IndexPersistence save throttling", () => { }); it("saves at most once per interval however often changes are scheduled", async () => { - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); - for (let i = 0; i < 50; i++) persistence.scheduleSave(); + for (let i = 0; i < 50; i++) { + touch(vector, `obs_${i}`); + persistence.scheduleSave(); + } await vi.advanceTimersByTimeAsync(59_000); - expect(manifestSaves(kv, "data:manifest")).toBe(0); + expect(metaSaves(kv)).toBe(0); await vi.advanceTimersByTimeAsync(1_000); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); - for (let i = 0; i < 50; i++) persistence.scheduleSave(); + for (let i = 0; i < 50; i++) { + touch(vector, `obs_more_${i}`); + persistence.scheduleSave(); + } await vi.advanceTimersByTimeAsync(30_000); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); await vi.advanceTimersByTimeAsync(30_000); - expect(manifestSaves(kv, "data:manifest")).toBe(2); + expect(metaSaves(kv)).toBe(2); }); it("an explicit save runs immediately and cancels the pending timer", async () => { - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); + touch(vector, "obs_1"); persistence.scheduleSave(); await persistence.save(); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); await vi.advanceTimersByTimeAsync(120_000); - expect(manifestSaves(kv, "data:manifest")).toBe(1); + expect(metaSaves(kv)).toBe(1); }); it("never runs two saves at once and coalesces requests made during a save into one", async () => { @@ -108,11 +99,12 @@ describe("IndexPersistence save throttling", () => { return result; }, }; - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + touch(vector, "obs_1"); + const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); const first = persistence.save(); + touch(vector, "obs_2"); const second = persistence.save(); const third = persistence.save(); expect(second).toBe(third); @@ -122,42 +114,12 @@ describe("IndexPersistence save throttling", () => { release(); await Promise.all([first, second, third]); - expect(manifestSaves(kv, "data:manifest")).toBe(2); + expect(metaSaves(kv)).toBe(2); expect(maxInFlight).toBe(1); expect(persistence.status().saving).toBe(false); }); - it("a failing BM25 leg does not stop the vector leg from saving", async () => { - const failingKv = { - ...kv, - set: async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === "data:manifest") throw new Error("bm25 manifest write failed"); - return kv.set(scope, key, data); - }, - }; - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const vector = new VectorIndex(); - vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); - const persistence = new IndexPersistence(failingKv as never, bm25, vector, { saveIntervalMs: 60_000 }); - - persistence.scheduleSave(); - await persistence.save(); - - const status = persistence.status(); - expect(status.bm25.lastError).toBe("bm25 manifest write failed"); - expect(status.bm25.dirtySince).not.toBeNull(); - expect(status.vector?.lastError).toBeNull(); - expect(status.vector?.lastSavedAt).not.toBeNull(); - expect(status.vector?.dirtySince).toBeNull(); - expect(status.vector?.serializedChars).toBeGreaterThan(0); - - const loaded = await new IndexPersistence(kv as never, new SearchIndex(), null).load(); - expect(loaded.bm25).toBeNull(); - expect(loaded.vector?.size).toBe(1); - }); - - it("keeps a leg dirty when a change arrives while it is being saved", async () => { + it("keeps the index dirty when a change arrives while it is being saved", async () => { let release: () => void = () => undefined; const gate = new Promise((resolve) => { release = resolve; @@ -173,63 +135,31 @@ describe("IndexPersistence save throttling", () => { return kv.set(scope, key, data); }, }; - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(slowKv as never, bm25, null, { saveIntervalMs: 60_000 }); + const vector = new VectorIndex(); + touch(vector, "obs_1"); + const persistence = new IndexPersistence(slowKv as never, vector, { saveIntervalMs: 60_000, bucketSize: 16 }); const saving = persistence.save(); + touch(vector, "obs_2"); persistence.scheduleSave(); release(); await saving; - expect(persistence.status().bm25.lastSavedAt).not.toBeNull(); - expect(persistence.status().bm25.dirtySince).not.toBeNull(); + expect(persistence.status().vector?.lastSavedAt).not.toBeNull(); + expect(persistence.status().vector?.dirtySince).not.toBeNull(); await persistence.save(); - expect(persistence.status().bm25.dirtySince).toBeNull(); - }); - - it("clears vector dirtySince when a change arrives during the bm25 leg's write", async () => { - let release: () => void = () => undefined; - const gate = new Promise((resolve) => { - release = resolve; - }); - let heldBm25Manifest = false; - const slowKv = { - ...kv, - set: async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === "data:manifest" && !heldBm25Manifest) { - heldBm25Manifest = true; - await gate; - } - return kv.set(scope, key, data); - }, - }; - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const vector = new VectorIndex(); - vector.add("obs_1", "ses_1", new Float32Array([0.1, 0.2, 0.3])); - const persistence = new IndexPersistence(slowKv as never, bm25, vector, { saveIntervalMs: 60_000 }); - - const saving = persistence.save(); - persistence.scheduleSave(); - release(); - await saving; - - const status = persistence.status(); - expect(status.bm25.dirtySince).not.toBeNull(); - expect(status.vector?.dirtySince).toBeNull(); - expect(status.vector?.lastSavedAt).not.toBeNull(); + expect(persistence.status().vector?.dirtySince).toBeNull(); }); it("stop prevents later scheduled saves", async () => { - const bm25 = new SearchIndex(); - bm25.add(obs("obs_1", "alpha")); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 1_000 }); + const vector = new VectorIndex(); + touch(vector, "obs_1"); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 1_000, bucketSize: 16 }); persistence.stop(); persistence.scheduleSave(); await vi.advanceTimersByTimeAsync(10_000); - expect(manifestSaves(kv, "data:manifest")).toBe(0); + expect(metaSaves(kv)).toBe(0); }); }); @@ -246,7 +176,15 @@ function statusInputs(overrides: Partial = {}): StatusInputs { provider: "llm", embeddingProvider: "embeddings", flags: [], - index: { bm25Documents: 1, vectorDocuments: 1, observationsIndexed: 1, missingObservations: 0, sessions: 1 }, + index: { + bm25Documents: 1, + vectorDocuments: 1, + observationsIndexed: 1, + missingObservations: 0, + sessions: 1, + bm25Incomplete: false, + pendingVectorBackfill: 0, + }, graph: null, graphExtractionEnabled: false, ...overrides, @@ -258,78 +196,53 @@ const cleanLeg = { lastError: null, lastErrorAt: null, dirtySince: null, - serializedChars: 1000, }; +function persistenceStatus(vector: typeof cleanLeg | null, pendingChanges = 0) { + return { saveIntervalMs: 600_000, saving: false, buckets: 256, pendingChanges, vector, vectorCountShortfall: null }; +} + describe("status reports index persistence", () => { - it("is quiet when both legs saved cleanly", () => { - const report = evaluateStatus( - statusInputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, bm25: cleanLeg, vector: cleanLeg } }), - ); + it("is quiet when the vector index saved cleanly", () => { + const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(cleanLeg) })); expect(report.problems).toEqual([]); - expect(report.indexPersistence?.vector?.serializedChars).toBe(1000); + expect(report.indexPersistence?.buckets).toBe(256); }); - it("reports a failing leg as an error", () => { + it("reports a failing vector save as an error", () => { const report = evaluateStatus( statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: cleanLeg, - vector: { ...cleanLeg, lastError: "Invalid string length", lastErrorAt: "2026-09-25T11:59:30.000Z" }, - }, + indexPersistence: persistenceStatus({ + ...cleanLeg, + lastError: "3 of 10 vector writes failed: timed out", + lastErrorAt: "2026-09-25T11:59:30.000Z", + } as never), }), ); expect(report.status).toBe("error"); expect(report.problems.map((p) => p.code)).toEqual(["index-save-failing"]); - expect(report.problems[0].message).toContain("vector"); - }); - - it("warns when the vector index passes 80% of the Node.js string limit", () => { - const report = evaluateStatus( - statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: cleanLeg, - vector: { ...cleanLeg, serializedChars: Math.ceil(V8_MAX_STRING_CHARS * 0.85) }, - }, - }), - ); - expect(report.status).toBe("warn"); - expect(report.problems.map((p) => p.code)).toEqual(["vector-index-near-string-limit"]); - expect(report.problems[0].message).toContain("85%"); + expect(report.problems[0].message).toContain("3 of 10 vector writes failed"); }); it("warns when unsaved changes are older than twice the save interval", () => { const report = evaluateStatus( - statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" }, - vector: { ...cleanLeg, dirtySince: "2026-09-25T11:55:00.000Z" }, - }, - }), + statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:30:00.000Z" } as never, 4) }), ); expect(report.problems.map((p) => p.code)).toEqual(["index-save-stale"]); - expect(report.problems[0].message).toContain("BM25"); }); - it("shows save state and vector size on the status page", () => { + it("does not report vector persistence when vector search is off", () => { + const report = evaluateStatus(statusInputs({ indexPersistence: persistenceStatus(null) })); + expect(report.problems).toEqual([]); + }); + + it("shows save state and bucket storage on the status page", () => { const report = evaluateStatus( - statusInputs({ - indexPersistence: { - saveIntervalMs: 600_000, - saving: false, - bm25: { ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" }, - vector: { ...cleanLeg, serializedChars: 53_687_089 }, - }, - }), + statusInputs({ indexPersistence: persistenceStatus({ ...cleanLeg, dirtySince: "2026-09-25T11:59:50.000Z" } as never, 3) }), ); const html = renderStatusHtml(report, "n"); - expect(html).toContain(""); - expect(html).toContain("53,687,089 characters (10% of the Node.js string limit)"); + expect(html).toContain(""); + expect(html).toContain(""); + expect(html).toContain(""); }); }); diff --git a/test/index-persistence.test.ts b/test/index-persistence.test.ts index 802084bd2d..568d180006 100644 --- a/test/index-persistence.test.ts +++ b/test/index-persistence.test.ts @@ -1,740 +1,475 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { IndexPersistence } from "../src/state/index-persistence.js"; +import { IndexPersistence, vectorBucketScope } from "../src/state/index-persistence.js"; import { SearchIndex } from "../src/state/search-index.js"; import { VectorIndex } from "../src/state/vector-index.js"; import type { CompressedObservation } from "../src/types.js"; -const BM25_SCOPE = "mem:index:bm25"; -const BM25_LEGACY_KEY = "data"; -const BM25_MANIFEST_KEY = "data:manifest"; -const VECTOR_LEGACY_KEY = "vectors"; -const VECTOR_MANIFEST_KEY = "vectors:manifest"; - -type TestIndexShardManifest = { - v: 1; - generation?: string; - shards: Array<{ scope: string; key: string; chars: number }>; - chars: number; -}; +const INDEX_SCOPE = "mem:index:bm25"; +const META_KEY = "vectors:meta"; function mockKV() { const store = new Map>(); + const ops: Array<{ op: "set" | "delete" | "list"; scope: string; key?: string }> = []; return { - get: async (scope: string, key: string): Promise => { - return (store.get(scope)?.get(key) as T) ?? null; - }, + store, + ops, + get: async (scope: string, key: string): Promise => (store.get(scope)?.get(key) as T) ?? null, set: async (scope: string, key: string, data: T): Promise => { + ops.push({ op: "set", scope, key }); if (!store.has(scope)) store.set(scope, new Map()); store.get(scope)!.set(key, data); return data; }, delete: async (scope: string, key: string): Promise => { + ops.push({ op: "delete", scope, key }); store.get(scope)?.delete(key); }, list: async (scope: string): Promise => { - const entries = store.get(scope); - return entries ? (Array.from(entries.values()) as T[]) : []; + ops.push({ op: "list", scope }); + return Array.from(store.get(scope)?.values() ?? []) as T[]; }, }; } type MockKV = ReturnType; -function makeObs( - overrides: Partial = {}, -): CompressedObservation { +function vec(values: number[]): Float32Array { + return new Float32Array(values); +} + +function vectorWith(entries: Array<[string, number[]]>): VectorIndex { + const index = new VectorIndex(); + for (const [id, values] of entries) index.add(id, `ses_${id}`, vec(values)); + return index; +} + +function bucketSets(kv: MockKV): Array<{ scope: string; key?: string }> { + return kv.ops.filter((o) => o.op === "set" && o.scope.startsWith(`${INDEX_SCOPE}:vec:`)); +} + +function obs(id: string): CompressedObservation { return { - id: "obs_1", + id, sessionId: "ses_1", timestamp: new Date().toISOString(), type: "file_edit", - title: "Edit auth middleware", - subtitle: "JWT validation", - facts: ["Added token check"], - narrative: "Modified the auth middleware to validate JWT tokens", - concepts: ["authentication", "jwt"], - files: ["src/middleware/auth.ts"], - importance: 7, - ...overrides, + title: `title ${id}`, + facts: [], + narrative: `narrative ${id}`, + concepts: [], + files: [], + importance: 5, }; } -function makeBm25(id: string, title: string): SearchIndex { - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id, title, narrative: `${title} narrative` })); - return bm25; +async function writeLegacyVectorSnapshot( + kv: MockKV, + vector: VectorIndex, + options: { generation?: string; monolithic?: boolean; shardChars?: number } = {}, +): Promise { + const serialized = vector.serialize(); + if (options.monolithic) { + await kv.set(INDEX_SCOPE, "vectors", serialized); + return; + } + const generation = options.generation ?? "idx_mfabcd12_aaaaaaaaaaaa"; + const size = options.shardChars ?? 40; + const shards: Array<{ scope: string; key: string; chars: number }> = []; + for (let offset = 0, i = 0; offset < serialized.length; offset += size, i++) { + const scope = `${INDEX_SCOPE}:vectors:${generation}:${String(i).padStart(5, "0")}`; + const chunk = serialized.slice(offset, offset + size); + await kv.set(scope, "data", chunk); + shards.push({ scope, key: "data", chars: chunk.length }); + } + await kv.set(INDEX_SCOPE, "vectors:manifest", { v: 1, generation, shards, chars: serialized.length }); } -function makeVector(id = "obs_1"): VectorIndex { - const vector = new VectorIndex(); - vector.add(id, "ses_1", new Float32Array([0.1, 0.2, 0.3])); - return vector; +async function writeLegacyBm25Snapshot(kv: MockKV): Promise { + const bm25 = new SearchIndex(); + bm25.add(obs("obs_legacy")); + const serialized = bm25.serialize(); + const scope = `${INDEX_SCOPE}:bm25:idx_mfabcd12_bbbbbbbbbbbb:00000`; + await kv.set(scope, "data", serialized); + await kv.set(INDEX_SCOPE, "data:manifest", { + v: 1, + generation: "idx_mfabcd12_bbbbbbbbbbbb", + shards: [{ scope, key: "data", chars: serialized.length }], + chars: serialized.length, + }); } -async function getBm25Manifest(kv: MockKV): Promise { - const manifest = await kv.get( - BM25_SCOPE, - BM25_MANIFEST_KEY, - ); - expect(manifest).not.toBeNull(); - return manifest!; +function expectSameVectors(actual: VectorIndex | null, expected: VectorIndex): void { + expect(actual).not.toBeNull(); + expect(actual!.size).toBe(expected.size); + for (const [id, entry] of expected.entries()) { + const loaded = actual!.get(id); + expect(loaded?.sessionId).toBe(entry.sessionId); + expect(Array.from(loaded!.embedding)).toEqual(Array.from(entry.embedding)); + } } -describe("IndexPersistence", () => { - let kv: ReturnType; +describe("IndexPersistence bucketed vector storage", () => { + let kv: MockKV; beforeEach(() => { - vi.useFakeTimers(); kv = mockKV(); }); - afterEach(() => { - vi.useRealTimers(); - }); - - it("saves and loads BM25 index round-trip", async () => { - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); + it("fills buckets in insertion order and rolls to a new one at the cap", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ["obs_c", [0.7, 0.8, 0.9]], + ]); + await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save(); + + expect(kv.store.get(vectorBucketScope(0))?.has("obs_a")).toBe(true); + expect(kv.store.get(vectorBucketScope(0))?.has("obs_b")).toBe(true); + expect(kv.store.get(vectorBucketScope(1))?.has("obs_c")).toBe(true); + const meta = await kv.get<{ v: number; bucketCount: number; count: number }>(INDEX_SCOPE, META_KEY); + expect(meta).toMatchObject({ v: 3, bucketCount: 2, count: 3 }); + + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 }).load(); + expect(loaded.state).toBe("buckets"); + expectSameVectors(loaded.vector, vector); + expect(loaded.vector!.pendingChanges).toBe(0); + }); + + it("writes only the bucket entry of a single added vector", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); + await persistence.save(); + kv.ops.length = 0; - const persistence = new IndexPersistence(kv as never, bm25, null); + vector.add("obs_new", "ses_new", vec([1, 0, 0])); await persistence.save(); - const loaded = await persistence.load(); - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.size).toBe(1); - const results = loaded.bm25!.search("auth"); - expect(results.length).toBe(1); + const writes = bucketSets(kv); + expect(writes).toEqual([{ op: "set", scope: vectorBucketScope(0), key: "obs_new" }]); }); - it("saves BM25 index shards outside the BM25 metadata scope", async () => { - const bm25 = new SearchIndex(); - bm25.add( - makeObs({ - id: "obs_1", - title: "auth handler ".repeat(40), - narrative: "JWT middleware validation ".repeat(40), - }), - ); - - const persistence = new IndexPersistence(kv as never, bm25, null, { - shardChars: 80, - createGeneration: () => "gen_bm25", - }); + it("does not write when nothing changed", async () => { + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); await persistence.save(); + kv.ops.length = 0; - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_bm25"); - expect(manifest.shards.length).toBeGreaterThan(1); - expect(manifest.shards[0].scope).toContain(":gen_bm25:"); - await expect(kv.get(BM25_SCOPE, BM25_LEGACY_KEY)).resolves.toBeNull(); - await expect( - kv.get(manifest.shards[0].scope, manifest.shards[0].key), - ).resolves.toEqual(expect.any(String)); + await persistence.save(); - const loaded = await persistence.load(); - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.search("auth").length).toBe(1); + expect(kv.ops.filter((o) => o.op !== "list")).toEqual([]); }); - it("loads legacy monolithic BM25 and vector snapshots", async () => { - const bm25 = makeBm25("obs_1", "legacy auth handler"); - const vector = makeVector("obs_1"); - await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, bm25.serialize()); - await kv.set(BM25_SCOPE, VECTOR_LEGACY_KEY, vector.serialize()); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.search("legacy").length).toBe(1); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(1); + it("removes deleted vectors from their bucket", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); + await persistence.save(); + + vector.remove("obs_a"); + await persistence.save(); + + expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(0), key: "obs_a" }); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); + expect(loaded.vector!.has("obs_a")).toBe(false); + expect(loaded.vector!.has("obs_b")).toBe(true); }); - it("fails closed instead of falling back when manifest reads fail", async () => { - const legacy = makeBm25("obs_legacy", "legacy stale snapshot"); - await kv.set(BM25_SCOPE, BM25_LEGACY_KEY, legacy.serialize()); - const failingKv = { - ...kv, - get: vi.fn(async (scope: string, key: string): Promise => { - if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { - throw new Error("manifest backend unavailable"); - } - return kv.get(scope, key); - }), - }; + it("a delete touches only the bucket that owns the id, in a multi-bucket store", async () => { + const vector = vectorWith( + Array.from({ length: 5 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), + ); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 2 }); + await persistence.save(); + kv.ops.length = 0; - const loaded = await new IndexPersistence( - failingKv as never, - new SearchIndex(), - null, - ).load(); + vector.remove("obs_2"); + await persistence.save(); - expect(loaded.bm25).toBeNull(); + const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(1)])); + expect(kv.ops).toContainEqual({ op: "delete", scope: vectorBucketScope(1), key: "obs_2" }); }); - it("fails closed when legacy snapshot reads fail", async () => { - const failingKv = { - ...kv, - get: vi.fn(async (scope: string, key: string): Promise => { - if (scope === BM25_SCOPE && key === BM25_LEGACY_KEY) { - throw new Error("legacy backend unavailable"); - } - return kv.get(scope, key); - }), - }; + it("clearing the index deletes every persisted vector", async () => { + const vector = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 16 }); + await persistence.save(); - const loaded = await new IndexPersistence( - failingKv as never, - new SearchIndex(), - null, - ).load(); + vector.clear(); + vector.add("obs_c", "ses_c", vec([0, 1, 0])); + await persistence.save(); - expect(loaded.bm25).toBeNull(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); + expect([...loaded.vector!.entries()].map(([id]) => id)).toEqual(["obs_c"]); }); - it("loads sharded manifests that omit optional generation metadata", async () => { - const bm25 = makeBm25("obs_1", "deterministic shard auth"); - const serialized = bm25.serialize(); - const chunks = [serialized.slice(0, 50), serialized.slice(50)]; - await kv.set("mem:index:bm25:bm25:00000", "data", chunks[0]); - await kv.set("mem:index:bm25:bm25:00001", "data", chunks[1]); - await kv.set(BM25_SCOPE, BM25_MANIFEST_KEY, { - v: 1, - chars: serialized.length, - shards: [ - { - scope: "mem:index:bm25:bm25:00000", - key: "data", - chars: chunks[0].length, - }, - { - scope: "mem:index:bm25:bm25:00001", - key: "data", - chars: chunks[1].length, - }, - ], - }); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.search("deterministic").length).toBe(1); - }); + it("keeps failed writes pending and retries them on the next save", async () => { + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + let fail = true; + const flakyKv = { + ...kv, + set: async (scope: string, key: string, data: T): Promise => { + if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("state::set timed out"); + return kv.set(scope, key, data); + }, + }; + const persistence = new IndexPersistence(flakyKv as never, vector, { bucketSize: 16 }); - it("saves and loads vector index round-trip", async () => { - const bm25 = new SearchIndex(); - const vector = makeVector(); + await expect(persistence.save()).resolves.toBeUndefined(); + const failed = persistence.status(); + expect(failed.vector?.lastError).toBe("1 of 1 vector writes failed: state::set timed out"); + expect(failed.pendingChanges).toBe(1); + expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull(); - const persistence = new IndexPersistence(kv as never, bm25, vector); + fail = false; await persistence.save(); - - const loaded = await persistence.load(); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(1); + expect(persistence.status().vector?.lastError).toBeNull(); + expect(persistence.status().pendingChanges).toBe(0); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); + expectSameVectors(loaded.vector, vector); }); - it("saves vector index shards outside the BM25 scope", async () => { - const bm25 = new SearchIndex(); + it("a save after many vectors added over several waves touches only the currently open bucket", async () => { const vector = new VectorIndex(); - vector.add( - "obs_1", - "ses_1", - new Float32Array(Array.from({ length: 32 }, (_, i) => i)), - ); + const persistence = new IndexPersistence(kv as never, vector, { bucketSize: 100 }); - const persistence = new IndexPersistence(kv as never, bm25, vector, { - shardChars: 40, - createGeneration: () => "gen_vector", - }); + for (let i = 0; i < 950; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0])); await persistence.save(); + expect(kv.store.get(vectorBucketScope(9))?.size).toBe(50); + for (let bucket = 0; bucket <= 8; bucket++) expect(kv.store.get(vectorBucketScope(bucket))?.size).toBe(100); + kv.ops.length = 0; - const manifest = await kv.get( - BM25_SCOPE, - VECTOR_MANIFEST_KEY, - ); - expect(manifest).not.toBeNull(); - expect(manifest!.generation).toBe("gen_vector"); - expect(manifest!.shards.length).toBeGreaterThan(1); - expect(manifest!.shards[0].scope).toContain(":gen_vector:"); - await expect(kv.get(BM25_SCOPE, VECTOR_LEGACY_KEY)).resolves.toBeNull(); - await expect( - kv.get(manifest!.shards[0].scope, manifest!.shards[0].key), - ).resolves.toEqual(expect.any(String)); + for (let i = 950; i < 1000; i++) vector.add(`obs_${i}`, "ses_1", vec([i, 0, 0])); + await persistence.save(); + let touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(9)])); + expect(kv.store.get(vectorBucketScope(9))?.size).toBe(100); + kv.ops.length = 0; - const loaded = await persistence.load(); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(1); + vector.add("obs_1000", "ses_1", vec([1000, 0, 0])); + await persistence.save(); + touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(10)])); }); - it("persists empty vector snapshots so cleared vectors do not reload", async () => { - const previousBm25 = makeBm25("obs_old", "alpha previous snapshot"); - const previousVector = makeVector("obs_old"); - await new IndexPersistence(kv as never, previousBm25, previousVector, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - - const nextBm25 = makeBm25("obs_new", "bravo new snapshot"); - const emptyVector = new VectorIndex(); - await new IndexPersistence(kv as never, nextBm25, emptyVector, { - shardChars: 80, - createGeneration: () => "gen_empty", - }).save(); - - const vectorManifest = await kv.get( - BM25_SCOPE, - VECTOR_MANIFEST_KEY, + it("a smaller bucket size configured later opens a new bucket instead of moving existing data", async () => { + const vector = vectorWith( + Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), ); - expect(vectorManifest).not.toBeNull(); - expect(vectorManifest!.generation).toBe("gen_empty"); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); - expect(loaded.vector).not.toBeNull(); - expect(loaded.vector!.size).toBe(0); - }); + await new IndexPersistence(kv as never, vector, { bucketSize: 4 }).save(); + for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true); - it("avoids one oversized state::set string payload for persisted indexes", async () => { - const maxStringPayloadChars = 80; - const bm25 = new SearchIndex(); - bm25.add( - makeObs({ - id: "obs_1", - title: "large persisted snapshot ".repeat(40), - narrative: "oversized state set reproduction ".repeat(40), - }), - ); - const vector = new VectorIndex(); - vector.add( - "obs_1", - "ses_1", - new Float32Array(Array.from({ length: 64 }, (_, i) => i / 10)), - ); - const guardedKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if ( - typeof data === "string" && - data.length > maxStringPayloadChars - ) { - throw new Error(`oversized state::set payload: ${scope}/${key}`); - } - return kv.set(scope, key, data); - }), - }; + const reloaded = new VectorIndex(); + const persistence = new IndexPersistence(kv as never, reloaded, { bucketSize: 2 }); + const loaded = await persistence.load(); + reloaded.restoreFrom(loaded.vector!); + kv.ops.length = 0; + + reloaded.add("obs_new", "ses_new", vec([9, 9, 9])); + await persistence.save(); - await new IndexPersistence(guardedKv as never, bm25, vector, { - shardChars: maxStringPayloadChars, - createGeneration: () => "gen_payload_limit", - }).save(); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("oversized").length).toBe(1); - expect(loaded.vector!.size).toBe(1); + const touched = new Set(kv.ops.filter((o) => o.scope.startsWith(`${INDEX_SCOPE}:vec:`)).map((o) => o.scope)); + expect(touched).toEqual(new Set([vectorBucketScope(1)])); + for (let i = 0; i < 4; i++) expect(kv.store.get(vectorBucketScope(0))?.has(`obs_${i}`)).toBe(true); }); - it("falls back to the default shard size for fractional values below one", async () => { - const bm25 = makeBm25("obs_fraction", "fractional shard config"); - let newShardWrites = 0; - const guardedKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope.includes(":gen_fraction:")) { - newShardWrites += 1; - if (newShardWrites > 3) { - throw new Error("fractional shard size caused zero-width shards"); - } - } - return kv.set(scope, key, data); - }), - }; + it("detects a partial bucket set against the saved count and reports it in status", async () => { + const vector = vectorWith( + Array.from({ length: 4 }, (_, i) => [`obs_${i}`, [i, i + 1, i + 2]] as [string, number[]]), + ); + await new IndexPersistence(kv as never, vector, { bucketSize: 2 }).save(); + kv.store.get(vectorBucketScope(1))?.clear(); - await new IndexPersistence(guardedKv as never, bm25, null, { - shardChars: 0.5, - createGeneration: () => "gen_fraction", - }).save(); + const persistence = new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 2 }); + const loaded = await persistence.load(); - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_fraction"); - expect(manifest.shards.length).toBe(1); - expect(newShardWrites).toBe(1); + expect(loaded.state).toBe("buckets"); + expect(loaded.vector!.size).toBe(2); + expect(loaded.expectedCount).toBe(4); + expect(persistence.status().vectorCountShortfall).toEqual({ expected: 4, loaded: 2 }); }); - it("keeps the previous generation when a shard write fails before manifest commit", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - const previousManifest = await getBm25Manifest(kv); - - let newShardWrites = 0; + it("reports storage as unavailable when the metadata read fails", async () => { const failingKv = { ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope.includes(":gen_new:")) { - newShardWrites += 1; - if (newShardWrites === 2) throw new Error("shard write failed"); - } - return kv.set(scope, key, data); - }), + get: async () => { + throw new Error("engine down"); + }, }; + const loaded = await new IndexPersistence(failingKv as never, new VectorIndex(), { bucketSize: 16 }).load(); + expect(loaded).toEqual({ vector: null, state: "unavailable", savedAt: null }); + }); - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual( - previousManifest, - ); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("alpha").length).toBe(1); - expect(loaded.bm25!.search("bravo").length).toBe(0); + it("treats a missing store as empty, including adapters that return undefined", async () => { + const undefinedKv = { ...kv, get: async () => undefined }; + const loaded = await new IndexPersistence(undefinedKv as never, new VectorIndex(), { bucketSize: 16 }).load(); + expect(loaded).toEqual({ vector: null, state: "none", savedAt: null }); }); +}); - it("keeps the previous generation when manifest set rejects before commit", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - const previousManifest = await getBm25Manifest(kv); +describe("IndexPersistence migration from the single-string format", () => { + let kv: MockKV; - const failingKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { - throw new Error("manifest write failed"); - } - return kv.set(scope, key, data); - }), - }; + beforeEach(() => { + kv = mockKV(); + }); - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); + it("migrates a sharded vector snapshot into buckets and removes the old shards", async () => { + const legacy = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ["obs_c", [0.7, 0.8, 0.9]], + ]); + await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_aaaaaaaaaaaa" }); + await writeLegacyBm25Snapshot(kv); + + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); + + expect(loaded.state).toBe("migrated"); + expect(loaded.savedAt).toBe(new Date(parseInt("mfabcd12", 36)).toISOString()); + expectSameVectors(loaded.vector, legacy); + expect(loaded.vector!.pendingChanges).toBe(0); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull(); + expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull(); + for (const [scope, entries] of kv.store) { + if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0); + } - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual( - previousManifest, - ); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("alpha").length).toBe(1); - expect(loaded.bm25!.search("bravo").length).toBe(0); + const reloaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); + expect(reloaded.state).toBe("buckets"); + expectSameVectors(reloaded.vector, legacy); }); - it("keeps a generation loadable when manifest set commits before rejecting", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); + it("removes the old BM25 shards when missing keys read back as undefined, as the engine SDK returns them", async () => { + const baseGet = kv.get; + kv.get = (async (scope: string, key: string) => { + const value = await baseGet(scope, key); + return value === null ? undefined : value; + }) as typeof kv.get; + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy, { generation: "idx_mfabcd12_bbbbbbbbbbbb" }); + await writeLegacyBm25Snapshot(kv); - const failingKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === BM25_MANIFEST_KEY) { - await kv.set(scope, key, data); - throw new Error("manifest write timed out after commit"); - } - return kv.set(scope, key, data); - }), - }; + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_new"); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toEqual(expect.any(String)); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); + expect(loaded.state).toBe("migrated"); + expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeUndefined(); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeUndefined(); + for (const [scope, entries] of kv.store) { + if (scope.includes(":vectors:") || scope.includes(":bm25:idx_")) expect(entries.size).toBe(0); + } }); - it("deletes a shard that committed before set rejected", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); - const previousManifest = await getBm25Manifest(kv); + it("migrates a monolithic vector snapshot", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy, { monolithic: true }); - const failingKv = { - ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === "mem:index:bm25:bm25:gen_new:00000") { - await kv.set(scope, key, data); - throw new Error("state::set timed out after commit"); - } - return kv.set(scope, key, data); - }), - }; - - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(failingKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toEqual( - previousManifest, - ); - await expect( - kv.get("mem:index:bm25:bm25:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("alpha").length).toBe(1); - expect(loaded.bm25!.search("bravo").length).toBe(0); + expect(loaded.state).toBe("migrated"); + expectSameVectors(loaded.vector, legacy); + expect(await kv.get(INDEX_SCOPE, "vectors")).toBeNull(); }); - it("loads the new generation even when old generation cleanup fails", async () => { - const previous = makeBm25("obs_old", "alpha previous snapshot"); - await new IndexPersistence(kv as never, previous, null, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); + it("leaves an unreadable legacy snapshot in place and writes nothing", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy); + const manifest = await kv.get<{ shards: Array<{ scope: string; key: string }> }>(INDEX_SCOPE, "vectors:manifest"); + await kv.delete(manifest!.shards[0].scope, manifest!.shards[0].key); + kv.ops.length = 0; - const cleanupKv = { - ...kv, - delete: vi.fn(async () => { - throw new Error("cleanup failed"); - }), - }; - const next = makeBm25("obs_new", "bravo new snapshot"); - await new IndexPersistence(cleanupKv as never, next, null, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - const manifest = await getBm25Manifest(kv); - expect(manifest.generation).toBe("gen_new"); - expect(cleanupKv.delete).toHaveBeenCalled(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); - expect(loaded.bm25!.search("alpha").length).toBe(0); - }); + const loaded = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); - it("keeps the previous vector generation when vector save fails after BM25 publish", async () => { - const previousBm25 = makeBm25("obs_old", "alpha previous snapshot"); - const previousVector = makeVector("obs_old"); - await new IndexPersistence(kv as never, previousBm25, previousVector, { - shardChars: 80, - createGeneration: () => "gen_old", - }).save(); + expect(loaded.state).toBe("unavailable"); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); + expect(kv.ops.filter((o) => o.op === "set" || o.op === "delete")).toEqual([]); + }); - const failingKv = { + it("keeps the legacy snapshot when migration writes fail and finishes on a later save", async () => { + const legacy = vectorWith([ + ["obs_a", [0.1, 0.2, 0.3]], + ["obs_b", [0.4, 0.5, 0.6]], + ]); + await writeLegacyVectorSnapshot(kv, legacy); + let fail = true; + const flakyKv = { ...kv, - set: vi.fn(async (scope: string, key: string, data: T): Promise => { - if (scope === BM25_SCOPE && key === VECTOR_MANIFEST_KEY) { - throw new Error("vector manifest write failed"); - } + set: async (scope: string, key: string, data: T): Promise => { + if (fail && scope.startsWith(`${INDEX_SCOPE}:vec:`)) throw new Error("write failed"); return kv.set(scope, key, data); - }), + }, }; - const nextBm25 = makeBm25("obs_new", "bravo new snapshot"); - const nextVector = new VectorIndex(); - nextVector.add("obs_new", "ses_1", new Float32Array([0.4, 0.5, 0.6])); - - await new IndexPersistence(failingKv as never, nextBm25, nextVector, { - shardChars: 80, - createGeneration: () => "gen_new", - }).save(); - - await expect( - kv.get("mem:index:bm25:vectors:gen_new:00000", "data"), - ).resolves.toBeNull(); - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - expect(loaded.bm25!.search("bravo").length).toBe(1); - expect(loaded.vector!.size).toBe(1); - expect( - loaded.vector!.search(new Float32Array([0.1, 0.2, 0.3]))[0]?.obsId, - ).toBe("obs_old"); - }); - - it("fails closed when a manifest shard is missing", async () => { - const bm25 = makeBm25("obs_1", "alpha sharded snapshot"); - await new IndexPersistence(kv as never, bm25, null, { - shardChars: 80, - createGeneration: () => "gen_missing", - }).save(); - const manifest = await getBm25Manifest(kv); - await kv.delete(manifest.shards[0].scope, manifest.shards[0].key); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).toBeNull(); - }); - - it("fails closed when a manifest shard length mismatches", async () => { - const bm25 = makeBm25("obs_1", "alpha sharded snapshot"); - await new IndexPersistence(kv as never, bm25, null, { - shardChars: 80, - createGeneration: () => "gen_mismatch", - }).save(); - const manifest = await getBm25Manifest(kv); - const firstShard = manifest.shards[0]; - const chunk = await kv.get(firstShard.scope, firstShard.key); - await kv.set(firstShard.scope, firstShard.key, `${chunk}x`); - - const loaded = await new IndexPersistence( - kv as never, - new SearchIndex(), - null, - ).load(); - - expect(loaded.bm25).toBeNull(); - }); - it("fails closed before reading invalid shard descriptors", async () => { - await kv.set(BM25_SCOPE, BM25_MANIFEST_KEY, { - v: 1, - chars: 10, - shards: [{ scope: "", key: "data", chars: 10 }], - }); - const guardedKv = { - ...kv, - get: vi.fn(async (scope: string, key: string): Promise => { - if (scope === "") { - throw new Error("invalid shard descriptor was read"); - } - return kv.get(scope, key); - }), - }; + const live = new VectorIndex(); + const persistence = new IndexPersistence(flakyKv as never, live, { bucketSize: 16 }); + const loaded = await persistence.load(); + expect(loaded.state).toBe("unavailable"); + expectSameVectors(loaded.vector, legacy); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); + expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull(); - const loaded = await new IndexPersistence( - guardedKv as never, - new SearchIndex(), - null, - ).load(); + live.restoreFrom(loaded.vector!); + fail = false; + await persistence.save(); - expect(loaded.bm25).toBeNull(); - expect(guardedKv.get).not.toHaveBeenCalledWith("", "data"); + const next = await new IndexPersistence(kv as never, new VectorIndex(), { bucketSize: 16 }).load(); + expect(next.state).toBe("buckets"); + expectSameVectors(next.vector, legacy); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).toBeNull(); }); - it("scheduleSave debounces multiple calls", async () => { - const bm25 = new SearchIndex(); - const persistence = new IndexPersistence(kv as never, bm25, null, { saveIntervalMs: 5000 }); + it("without vector search, removes only the old BM25 snapshot", async () => { + const legacy = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + await writeLegacyVectorSnapshot(kv, legacy); + await writeLegacyBm25Snapshot(kv); - persistence.scheduleSave(); - persistence.scheduleSave(); - persistence.scheduleSave(); + const loaded = await new IndexPersistence(kv as never, null, { bucketSize: 16 }).load(); - await expect(kv.get(BM25_SCOPE, BM25_MANIFEST_KEY)).resolves.toBeNull(); - - vi.advanceTimersByTime(5000); - await vi.runAllTimersAsync(); - - const saved = await kv.get(BM25_SCOPE, BM25_MANIFEST_KEY); - expect(saved).not.toBeNull(); + expect(loaded).toEqual({ vector: null, state: "none", savedAt: null }); + expect(await kv.get(INDEX_SCOPE, "data:manifest")).toBeNull(); + expect(await kv.get(INDEX_SCOPE, "vectors:manifest")).not.toBeNull(); }); +}); - it("stop clears the pending timer", async () => { - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(kv as never, bm25, null); - - persistence.scheduleSave(); - persistence.stop(); +describe("IndexPersistence scheduling", () => { + let kv: MockKV; - vi.advanceTimersByTime(10000); - const saved = await kv.get(BM25_SCOPE, BM25_MANIFEST_KEY); - expect(saved).toBeNull(); + beforeEach(() => { + vi.useFakeTimers(); + kv = mockKV(); }); - it("returns null indexes when nothing has been saved", async () => { - const bm25 = new SearchIndex(); - const persistence = new IndexPersistence(kv as never, bm25, null); - - const loaded = await persistence.load(); - expect(loaded.bm25).toBeNull(); - expect(loaded.vector).toBeNull(); + afterEach(() => { + vi.useRealTimers(); }); - it("scheduled save swallows kv.set rejection without unhandledRejection (#204)", async () => { + it("scheduled saves never raise unhandled rejections", async () => { const failingKv = { - ...mockKV(), + ...kv, set: vi.fn(async () => { - const err = new Error( - "TIMEOUT: invocation timed out after 30000ms", - ) as Error & { code?: string; function_id?: string }; + const err = new Error("TIMEOUT: invocation timed out after 30000ms") as Error & { code?: string }; err.code = "TIMEOUT"; - err.function_id = "state::set"; throw err; }), }; - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); - + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + const persistence = new IndexPersistence(failingKv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 }); let unhandled = false; const onUnhandled = () => { unhandled = true; }; process.on("unhandledRejection", onUnhandled); - try { persistence.scheduleSave(); - vi.advanceTimersByTime(5000); - await vi.runAllTimersAsync(); - // give microtasks a chance to flush - await Promise.resolve(); + await vi.advanceTimersByTimeAsync(5000); expect(failingKv.set).toHaveBeenCalled(); expect(unhandled).toBe(false); } finally { @@ -742,143 +477,12 @@ describe("IndexPersistence", () => { } }); - it("save() does not throw when kv.set rejects (#204)", async () => { - const failingKv = { - ...mockKV(), - set: vi.fn(async () => { - throw new Error("TIMEOUT"); - }), - }; - const bm25 = new SearchIndex(); - bm25.add(makeObs({ id: "obs_1", title: "auth handler" })); - const persistence = new IndexPersistence(failingKv as never, bm25, null, { saveIntervalMs: 5000 }); - - await expect(persistence.save()).resolves.toBeUndefined(); - }); - - // #797: first run after upgrading to 0.9.25 crashed with - // 'TypeError: Cannot read properties of undefined (reading "v")' - // because some iii-state adapters return `undefined` (not `null`) - // for a missing key. The load path's `value !== null` check passed - // undefined to loadManifestData, which then read `undefined.v`. - it("load() returns null instead of crashing when kv.get returns undefined for the manifest (#797)", async () => { - const undefinedKv = { - ...mockKV(), - get: vi.fn(async () => undefined), - }; - const persistence = new IndexPersistence( - undefinedKv as never, - new SearchIndex(), - null, - ); - - const loaded = await persistence.load(); - expect(loaded.bm25).toBeNull(); - expect(loaded.vector).toBeNull(); - }); - - it("load() does not crash when a manifest row value is the wrong shape (#797)", async () => { - const wrongShapeKv = { - ...mockKV(), - get: vi.fn(async () => "not-a-manifest"), - }; - const persistence = new IndexPersistence( - wrongShapeKv as never, - new SearchIndex(), - null, - ); - - await expect(persistence.load()).resolves.toBeDefined(); - }); -}); - -describe("index_persist audit gating", () => { - let kv: ReturnType; - let previousFlag: string | undefined; - - beforeEach(() => { - // AGENTMEMORY_* variables are documented as living in - // ~/.agentmemory/.env, so a developer running the suite on a - // configured machine can inherit this one. Clear it going in and put - // whatever was there back on the way out. - previousFlag = process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; - delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; - vi.useFakeTimers(); - kv = mockKV(); - }); - - afterEach(() => { - vi.useRealTimers(); - if (previousFlag === undefined) { - delete process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST; - } else { - process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = previousFlag; - } - }); - - async function indexPersistEntries(): Promise> { - const entries = await kv.list<{ operation: string }>("mem:audit"); - return entries.filter((entry) => entry.operation === "index_persist"); - } - - it("writes no index_persist audit entries by default", async () => { - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - expect(await indexPersistEntries()).toEqual([]); - }); - - it.each(["1", " 1 ", "true", "TRUE", " true "])( - "writes index_persist audit entries when set to %j", - async (value) => { - process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value; - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - expect((await indexPersistEntries()).length).toBeGreaterThan(0); - }, - ); - - // Anything that is not an affirmative stays off. "0" and "false" are the - // ones an operator is likely to reach for to disable it, and they must - // not read as "present, therefore enabled". - it.each(["0", "false", "yes", "", " "])( - "keeps auditing off when set to %j", - async (value) => { - process.env.AGENTMEMORY_AUDIT_INDEX_PERSIST = value; - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - expect(await indexPersistEntries()).toEqual([]); - }, - ); - - it("still persists the index when auditing is off", async () => { - const persistence = new IndexPersistence( - kv as never, - makeBm25("obs_1", "auth handler"), - null, - ); - - await persistence.save(); - - const loaded = await persistence.load(); - expect(loaded.bm25).not.toBeNull(); - expect(loaded.bm25!.size).toBe(1); + it("stop clears the pending timer", async () => { + const vector = vectorWith([["obs_a", [0.1, 0.2, 0.3]]]); + const persistence = new IndexPersistence(kv as never, vector, { saveIntervalMs: 5000, bucketSize: 16 }); + persistence.scheduleSave(); + persistence.stop(); + await vi.advanceTimersByTimeAsync(10_000); + expect(await kv.get(INDEX_SCOPE, META_KEY)).toBeNull(); }); }); diff --git a/test/search.test.ts b/test/search.test.ts index 84b8134e3a..99b7f4d6d3 100644 --- a/test/search.test.ts +++ b/test/search.test.ts @@ -209,4 +209,21 @@ describe("mem::search", () => { setVectorIndex(null); setEmbeddingProvider(null); }); + + it("a cold-start search rebuilds BM25 without touching the persisted vector index", async () => { + const vector = new VectorIndex(); + vector.add("obs_persisted", "ses_1", new Float32Array([0.1, 0.2, 0.3])); + setVectorIndex(vector); + getSearchIndex().clear(); + + const result = (await sdk.trigger("mem::search", { + query: "auth middleware", + })) as { results: Array<{ observation: CompressedObservation }> }; + + expect(result.results[0]?.observation.id).toBe("obs_a"); + expect(getVectorIndex()?.has("obs_persisted")).toBe(true); + expect(getVectorIndex()?.size).toBe(1); + + setVectorIndex(null); + }); }); diff --git a/test/status.test.ts b/test/status.test.ts index 6ecfb50ca5..2160a65f92 100644 --- a/test/status.test.ts +++ b/test/status.test.ts @@ -27,6 +27,8 @@ function inputs(overrides: Partial = {}): StatusInputs { observationsIndexed: 110, missingObservations: 0, sessions: 95, + bm25Incomplete: false, + pendingVectorBackfill: 0, }, graph: { totalNodes: 55, @@ -93,6 +95,28 @@ describe("evaluateStatus", () => { expect(report.status).toBe("info"); }); + it("reports an incomplete BM25 rebuild as an error", () => { + const report = evaluateStatus(inputs({ index: { ...inputs().index, bm25Incomplete: true } })); + expect(report.status).toBe("error"); + expect(codes(report)).toContain("bm25-rebuild-incomplete"); + }); + + it("notes a pending vector backfill without raising the overall status", () => { + const report = evaluateStatus(inputs({ index: { ...inputs().index, pendingVectorBackfill: 42 } })); + expect(report.status).toBe("info"); + const problem = report.problems.find((p) => p.code === "index-vector-backfill-pending"); + expect(problem?.message).toBe("42 documents are waiting for a vector embedding."); + }); + + it("reports a vector count shortfall from the last save as a warning", () => { + const report = evaluateStatus( + inputs({ indexPersistence: { saveIntervalMs: 600_000, saving: false, buckets: 3, pendingChanges: 0, vector: null, vectorCountShortfall: { expected: 100, loaded: 40 } } }), + ); + expect(report.status).toBe("warn"); + const problem = report.problems.find((p) => p.code === "index-vector-count-shortfall"); + expect(problem?.message).toContain("40 of 100 vectors"); + }); + it("checks snapshot presence when extraction is on, and snapshot age whenever a snapshot exists", () => { const noSnapshot = { totalNodes: 0, totalEdges: 0, fromSnapshot: false }; expect(codes(evaluateStatus(inputs({ graph: noSnapshot })))).toEqual(["graph-no-snapshot"]);
BM25 savesaved 1m ago, unsaved changes pendingBM25 indexrebuilt from stored content at bootVector savesaved 1m ago, unsaved changes pendingVector storage256 buckets, 3 unsaved changes